fix(knowledge): cut bounded text with a shared surrogate-safe helper inside the limit and leave stored upload metadata untouched

Author: waleedlatif1

apps/sim/lib/api/contracts/knowledge/documents.test.ts

@@ -271,7 +271,11 @@ describe('document filename and tag bounds', () => {
   })
 
   it('rejects a filename over the limit on create, upsert, and update with a descriptive message', () => {
-    for (const schema of [createDocumentBodySchema, upsertDocumentBodySchema, updateDocumentBodySchema]) {
+    for (const schema of [
+      createDocumentBodySchema,
+      upsertDocumentBodySchema,
+      updateDocumentBodySchema,
+    ]) {
       const result = schema.safeParse({ ...base, filename: overLimit })
       expect(result.success).toBe(false)
       expect(result.error?.issues[0]?.message).toBe(

apps/sim/lib/api/contracts/knowledge/documents.ts

@@ -122,13 +122,19 @@ export function parseDocumentTagFiltersParam(
 /** A text tag value that fits its index row; see {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */
 const documentTagValueSchema = z
   .string()
-  .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`)
+  .max(
+    MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+    `Tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+  )
 
 export const createDocumentBodySchema = z.object({
   filename: z
     .string()
     .min(1, 'Filename is required')
-    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`),
+    .max(
+      MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+      `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+    ),
   fileUrl: knowledgeDocumentFileUrlSchema,
   fileSize: z.number().min(1, 'File size must be greater than 0'),
   mimeType: z.string().min(1, 'MIME type is required'),
@@ -180,7 +186,10 @@ export const upsertDocumentBodySchema = z.object({
   filename: z
     .string()
     .min(1, 'Filename is required')
-    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`),
+    .max(
+      MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+      `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+    ),
   fileUrl: knowledgeDocumentFileUrlSchema,
   fileSize: z.number().min(1, 'File size must be greater than 0'),
   mimeType: z.string().min(1, 'MIME type is required'),
@@ -214,7 +223,11 @@ export const updateDocumentBodySchema = z.object({
   filename: z
     .string()
     .min(1, 'Filename is required')
-    .max(MAX_DOCUMENT_INDEXED_TEXT_LENGTH, `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`).optional(),
+    .max(
+      MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
+      `Filename cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
+    )
+    .optional(),
   enabled: z.boolean().optional(),
   chunkCount: z.number().min(0).optional(),
   tokenCount: z.number().min(0).optional(),

apps/sim/lib/knowledge/connectors/sync-persistence.test.ts

@@ -30,7 +30,10 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({
 vi.mock('@/connectors/registry.server', () => ({
   CONNECTOR_REGISTRY: {
     fixture: {
-      mapTags: (metadata: Record<string, unknown>) => ({ label: metadata.label, owner: metadata.owner }),
+      mapTags: (metadata: Record<string, unknown>) => ({
+        label: metadata.label,
+        owner: metadata.owner,
+      }),
     },
   },
 }))
@@ -366,7 +369,8 @@ describe('persistSourceDocumentFailures', () => {
       priorByExternalId: new Map(),
     })
     const [rows] = dbChainMockFns.values.mock.calls[0] as [Array<{ filename: string }>]
-    expect(rows[0].filename).toBe(`${'x'.repeat(512)}...`)
+    expect(rows[0].filename).toBe(`${'x'.repeat(509)}...`)
+    expect(rows[0].filename.length).toBe(512)
   })
   it('refuses to commit a failure under a reclaimed lease', async () => {
     queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb' }])
@@ -442,7 +446,7 @@ describe('resolveTagMapping', () => {
       { label: 'y'.repeat(5000), owner: 'Purchasing' },
       { tagSlotMapping: { label: 'tag1', owner: 'tag2' } }
     )
-    expect(tags?.tag1).toBe(`${'y'.repeat(512)}...`)
+    expect(tags?.tag1).toBe(`${'y'.repeat(509)}...`)
     expect(tags?.tag2).toBe('Purchasing')
   })
 
@@ -452,6 +456,7 @@ describe('resolveTagMapping', () => {
       { label: '\u{1F600}'.repeat(600) },
       { tagSlotMapping: { label: 'tag1' } }
     )
-    expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(512)}...`)
+    expect(tags?.tag1).toBe(`${'\u{1F600}'.repeat(254)}...`)
+    expect(tags?.tag1?.length).toBeLessThanOrEqual(512)
   })
 })

apps/sim/lib/knowledge/connectors/sync-persistence.ts

@@ -3,6 +3,7 @@ import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/
 import { createLogger } from '@sim/logger'
 import { chunkArray } from '@sim/utils/helpers'
 import { generateId } from '@sim/utils/id'
+import { truncateAtCodePoint } from '@sim/utils/string'
 import { and, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm'
 import { getInternalApiBaseUrl } from '@/lib/core/utils/urls'
 import type { DbOrTx } from '@/lib/db/types'
@@ -18,6 +19,7 @@ import type { ConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/conn
 import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at'
 import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits'
 import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock'
+import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 import type { DocumentData } from '@/lib/knowledge/documents/service'
 import { enqueueKnowledgeStorageCleanup } from '@/lib/knowledge/documents/storage-cleanup'
 import {
@@ -26,7 +28,6 @@ import {
 } from '@/lib/knowledge/documents/storage-upload'
 import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key'
 import { getFileMetadataByKeys } from '@/lib/uploads/server/metadata'
-import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 import { CONNECTOR_REGISTRY } from '@/connectors/registry.server'
 import type { DocumentTags, ExternalDocument } from '@/connectors/types'
 
@@ -174,16 +175,20 @@ export async function persistDocumentAcls(
 
 const MAX_SAFE_TITLE_LENGTH = 200
 
+/** The suffix a cut value carries, counted inside {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH}. */
+const INDEXED_TEXT_CUT_SUFFIX = '...'
+
 /**
  * Source titles and mapped tag values are untrusted machine input with no caller to refuse them,
- * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} by code point, never inside a
- * surrogate pair.
+ * so they are cut to {@link MAX_DOCUMENT_INDEXED_TEXT_LENGTH} code units, suffix included, and
+ * never inside a surrogate pair. The result always passes the document APIs' own bound.
  */
 function boundIndexedText(value: string): string {
-  if (value.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value
-  const points = Array.from(value)
-  if (points.length <= MAX_DOCUMENT_INDEXED_TEXT_LENGTH) return value
-  return `${points.slice(0, MAX_DOCUMENT_INDEXED_TEXT_LENGTH).join('')}...`
+  return truncateAtCodePoint(
+    value,
+    MAX_DOCUMENT_INDEXED_TEXT_LENGTH - INDEXED_TEXT_CUT_SUFFIX.length,
+    INDEXED_TEXT_CUT_SUFFIX
+  )
 }
 
 function sanitizeStorageTitle(title: string): string {

apps/sim/lib/knowledge/upload-metadata.test.ts

@@ -7,19 +7,8 @@ import {
   knowledgeDocumentUploadMetadataSchema,
   persistedKnowledgeDocumentUploadMetadataSchema,
 } from '@/lib/knowledge/upload-metadata'
-import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 
 describe('knowledgeDocumentUploadMetadataSchema', () => {
-  it('rejects a tag value that would not fit its index row', () => {
-    const result = knowledgeDocumentUploadMetadataSchema.safeParse({
-      tag1: 'a'.repeat(MAX_DOCUMENT_INDEXED_TEXT_LENGTH + 1),
-    })
-    expect(result.success).toBe(false)
-    expect(result.error?.issues[0]?.message).toBe(
-      `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
-    )
-  })
-
   it('rejects a recipe outside the accepted set', () => {
     const result = knowledgeDocumentUploadMetadataSchema.safeParse({
       processingOptions: { recipe: 'totally-bogus-recipe' },

apps/sim/lib/knowledge/upload-metadata.ts

@@ -1,6 +1,5 @@
 import { z } from 'zod'
 import type { RecursiveRecipe } from '@/lib/chunkers/types'
-import { MAX_DOCUMENT_INDEXED_TEXT_LENGTH } from '@/lib/knowledge/constants'
 
 /**
  * Recipes the recursive chunker implements. Mirrors `RecursiveRecipe`; the
@@ -36,10 +35,7 @@ const LANGUAGE_TAG_SHAPE = /^[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*$/
 
 const knowledgeDocumentUploadTagSchema = z
   .string()
-  .max(
-    MAX_DOCUMENT_INDEXED_TEXT_LENGTH,
-    `Knowledge document tag values cannot exceed ${MAX_DOCUMENT_INDEXED_TEXT_LENGTH} characters`
-  )
+  .max(1000, 'Knowledge document tag values cannot exceed 1000 characters')
   .optional()
 
 const knowledgeDocumentUploadTagShape = {

packages/utils/src/string.test.ts

@@ -16,6 +16,7 @@ import {
   slugify,
   stripVersionSuffix,
   truncate,
+  truncateAtCodePoint,
 } from './string.js'
 
 describe('slugify', () => {
@@ -304,3 +305,19 @@ describe('hasRegexMetacharacter', () => {
     }
   })
 })
+
+describe('truncateAtCodePoint', () => {
+  it('returns the input untouched when it fits', () => {
+    expect(truncateAtCodePoint('ab😀cd', 10)).toBe('ab😀cd')
+  })
+
+  it('cuts like truncate when the cut lands between code points', () => {
+    expect(truncateAtCodePoint('ab😀cd', 4)).toBe('ab😀...')
+    expect(truncateAtCodePoint('hello world', 8, ' …')).toBe('hello wo …')
+  })
+
+  it('moves the cut back one unit rather than splitting a surrogate pair', () => {
+    expect(truncateAtCodePoint('ab😀cd', 3)).toBe('ab...')
+    expect(truncateAtCodePoint('😀'.repeat(3), 3)).toBe('😀...')
+  })
+})

packages/utils/src/string.ts

@@ -33,6 +33,21 @@ export function truncate(str: string, sliceLength: number, suffix = '...'): stri
   return str.length > sliceLength ? str.slice(0, sliceLength) + suffix : str
 }
 
+/**
+ * Like {@link truncate}, but never cuts inside a surrogate pair: when the code unit at the cut
+ * would split an astral character, the cut moves one unit earlier. Lengths are still counted in
+ * UTF-16 code units, so the result of a cut is at most `sliceLength + suffix.length` units.
+ *
+ * @example
+ * truncateAtCodePoint('ab😀cd', 3)   // 'ab...' (the cut at 3 would split the emoji)
+ * truncateAtCodePoint('ab😀cd', 4)   // 'ab😀...'
+ */
+export function truncateAtCodePoint(str: string, sliceLength: number, suffix = '...'): string {
+  if (str.length <= sliceLength) return str
+  const splitsPair = sliceLength > 0 && (str.charCodeAt(sliceLength - 1) & 0xfc00) === 0xd800
+  return str.slice(0, splitsPair ? sliceLength - 1 : sliceLength) + suffix
+}
+
 /**
  * Lowercases `value` into the `[a-z0-9-]` charset: every run of other characters
  * becomes one hyphen, and leading and trailing hyphens are dropped.