diff --git a/src/filesystem/__tests__/lib.test.ts b/src/filesystem/__tests__/lib.test.ts index 1da741f0b0..c48a4f0837 100644 --- a/src/filesystem/__tests__/lib.test.ts +++ b/src/filesystem/__tests__/lib.test.ts @@ -718,6 +718,26 @@ describe('Lib Functions', () => { }); describe('headFile', () => { + it('preserves UTF-8 characters split across read chunks', async () => { + const content = Buffer.concat([ + Buffer.from('a'.repeat(1023)), + Buffer.from('中\nsecond line', 'utf8'), + ]); + const mockFileHandle = { + read: vi.fn(async (target: Buffer, _offset: number, length: number, position: number) => { + const bytes = content.subarray(position, position + length); + bytes.copy(target, 0); + return { bytesRead: bytes.length }; + }), + close: vi.fn().mockResolvedValue(undefined), + } as any; + mockFs.open.mockResolvedValue(mockFileHandle); + + const result = await headFile('/test/file.txt', 1); + + expect(result).toBe('a'.repeat(1023) + '中'); + }); + it('opens file for reading', async () => { // Mock file handle with proper typing const mockFileHandle = { diff --git a/src/filesystem/lib.ts b/src/filesystem/lib.ts index a1c6f04b67..6abd109ea5 100644 --- a/src/filesystem/lib.ts +++ b/src/filesystem/lib.ts @@ -6,6 +6,7 @@ import { diffLines, createTwoFilesPatch } from 'diff'; import { minimatch } from 'minimatch'; import { normalizePath, expandHome } from './path-utils.js'; import { isPathWithinAllowedDirectories } from './path-validation.js'; +import { StringDecoder } from 'string_decoder'; // Global allowed directories - set by the main module let allowedDirectories: string[] = []; @@ -308,42 +309,24 @@ export async function tailFile(filePath: string, numLines: number): Promise 0 && linesFound < numLines) { + while (position > 0 && rawBytes.toString('binary').split('\n').length <= numLines) { const size = Math.min(CHUNK_SIZE, position); position -= size; const { bytesRead } = await fileHandle.read(chunk, 0, size, position); if (!bytesRead) break; - // Get the chunk as a string and prepend any remaining text from previous iteration - const readData = chunk.slice(0, bytesRead).toString('utf-8'); - const chunkText = readData + remainingText; - - // Split by newlines and count - const chunkLines = normalizeLineEndings(chunkText).split('\n'); - - // If this isn't the end of the file, the first line is likely incomplete - // Save it to prepend to the next chunk - if (position > 0) { - remainingText = chunkLines[0]; - chunkLines.shift(); // Remove the first (incomplete) line - } - - // Add lines to our result (up to the number we need) - for (let i = chunkLines.length - 1; i >= 0 && linesFound < numLines; i--) { - lines.unshift(chunkLines[i]); - linesFound++; - } + // Keep bytes intact while reading backwards. Decoding each chunk can + // split a multi-byte UTF-8 character at the chunk boundary. + rawBytes = Buffer.concat([chunk.slice(0, bytesRead), rawBytes]); } - - return lines.join('\n'); + const lines = normalizeLineEndings(rawBytes.toString('utf-8')).split('\n'); + return lines.slice(Math.max(0, lines.length - numLines)).join('\n'); } finally { await fileHandle.close(); } @@ -357,13 +340,14 @@ export async function headFile(filePath: string, numLines: number): Promise 0 && lines.length < numLines) { lines.push(buffer);