From 5745c0db9af5066b866d2c1cb210b0d379bf2655 Mon Sep 17 00:00:00 2001 From: Kaiyi Date: Tue, 8 Sep 2026 17:26:04 +0800 Subject: [PATCH] feat(media): convert HEIC images to JPEG with sips on macOS HEIC/HEIF is the default photo and screenshot format on Apple devices, but model providers only accept PNG, JPEG, GIF, and WebP, so every ingestion point refused it with a "convert it first" message. On macOS the system `sips` tool decodes HEIC with Apple's codecs, so the engine now converts HEIC to JPEG in-process before the existing format gate, and the converted image flows through the unchanged compression pipeline (resize, byte budget, region crops, originals): - a shared transcoder that only activates on macOS with a host process service, runs sips against a scratch directory, and returns null on any failure so callers keep their previous behavior - ReadMediaFile hands the file path straight to sips on the runtime that owns the file and notes the conversion for the model - the daemon-file image resolver (@file references and pasted uploads) converts and compresses before inlining - kap-server prompt attachments (inline base64, path, uploaded file) accept an image transcoder; the prompt and skill routes wire the macOS one in, keeping the HEIC original next to the compression caption - the TUI clipboard reader converts a copied HEIC file through sips on macOS instead of silently ignoring it Other platforms keep the existing conversion guidance. --- .changeset/heic-macos-sips.md | 5 + .../src/utils/clipboard/clipboard-image.ts | 81 ++++-- apps/kimi-code/src/utils/image/image-mime.ts | 19 ++ .../utils/clipboard/clipboard-image.test.ts | 117 ++++++++- .../test/utils/image/image-mime.test.ts | 30 ++- .../src/agent/media/heic-transcode.ts | 130 ++++++++++ .../src/agent/media/image-originals.ts | 2 + .../src/agent/media/mediaResolverService.ts | 41 ++- .../read-media-file/readMediaFileTool.ts | 47 +++- .../agent-core-v2/src/app/telemetry/events.ts | 23 ++ packages/agent-core-v2/src/index.ts | 10 + .../test/agent/media/fakeSips.ts | 73 ++++++ .../test/agent/media/heic-transcode.test.ts | 210 +++++++++++++++ .../test/agent/media/mediaResolver.test.ts | 79 +++++- .../test/agent/media/tools/read-media.test.ts | 81 +++++- packages/kap-server/src/lib/promptMedia.ts | 109 ++++++++ packages/kap-server/src/routes/prompts.ts | 9 + packages/kap-server/src/routes/skills.ts | 9 + packages/kap-server/test/prompts.test.ts | 241 ++++++++++++++++++ 19 files changed, 1279 insertions(+), 37 deletions(-) create mode 100644 .changeset/heic-macos-sips.md create mode 100644 packages/agent-core-v2/src/agent/media/heic-transcode.ts create mode 100644 packages/agent-core-v2/test/agent/media/fakeSips.ts create mode 100644 packages/agent-core-v2/test/agent/media/heic-transcode.test.ts diff --git a/.changeset/heic-macos-sips.md b/.changeset/heic-macos-sips.md new file mode 100644 index 00000000000..7015db80f64 --- /dev/null +++ b/.changeset/heic-macos-sips.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": minor +--- + +Read and attach HEIC/HEIF images on macOS; they are converted to JPEG automatically, including files pasted into the prompt. diff --git a/apps/kimi-code/src/utils/clipboard/clipboard-image.ts b/apps/kimi-code/src/utils/clipboard/clipboard-image.ts index 6aae761c449..3f494bb6bba 100644 --- a/apps/kimi-code/src/utils/clipboard/clipboard-image.ts +++ b/apps/kimi-code/src/utils/clipboard/clipboard-image.ts @@ -4,6 +4,9 @@ * kimi-core's LLM pipeline only accepts PNG/JPEG/GIF/WebP, and the * clipboard sources we query already emit those formats on supported * platforms — so we deliberately do not include a BMP→PNG converter. + * The one exception is a copied HEIC/HEIF *file* on macOS (iPhone photos + * and screenshots): it is converted to JPEG through the system `sips` + * before being pasted. Elsewhere such a file is still declined. * * Lookup order: * macOS file clipboard -> osascript/AppKit file URLs @@ -22,7 +25,7 @@ import { tmpdir } from 'node:os'; import { basename, isAbsolute, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { parseImageMeta } from '#/utils/image/image-mime'; +import { isHeicImage, parseImageMeta } from '#/utils/image/image-mime'; import { DEFAULT_LIST_TIMEOUT_MS, @@ -82,6 +85,13 @@ const VIDEO_MIME_BY_SUFFIX: Readonly> = Object.freeze({ const DEFAULT_READ_TIMEOUT_MS = 3000; const DEFAULT_POWERSHELL_TIMEOUT_MS = 5000; +const SIPS_TIMEOUT_MS = 15_000; + +/** Platform facts a clipboard file path needs to be turned into pasteable media. */ +interface PathReadContext { + readonly run: RunCommand; + readonly platform: NodeJS.Platform; +} const MACOS_FILE_PATH_SCRIPT = String.raw` ObjC.import('AppKit'); @@ -170,7 +180,7 @@ function splitClipboardPathLines(text: string): string[] { return lines; } -function readImagePath(path: string): ClipboardImage | null { +function readImagePath(path: string, ctx: PathReadContext): ClipboardImage | null { let stat: ReturnType; try { stat = statSync(path); @@ -187,11 +197,44 @@ function readImagePath(path: string): ClipboardImage | null { } if (bytes.length === 0) return null; - const meta = parseImageMeta(bytes); + let meta = parseImageMeta(bytes); + if (meta === null && ctx.platform === 'darwin' && isHeicImage(bytes)) { + const converted = convertHeicViaSips(path, ctx.run); + if (converted === null) return null; + bytes = converted; + meta = parseImageMeta(bytes); + } if (meta === null) return null; return { kind: 'image', bytes: new Uint8Array(bytes), mimeType: meta.mime }; } +/** + * macOS ships `sips`, which decodes HEIC with the system codecs. The JPEG + * lands in a temp file (sips has no stdout mode) that is removed on every + * exit path; any failure declines the paste instead of surfacing an error. + */ +function convertHeicViaSips(path: string, run: RunCommand): Buffer | null { + const target = join(tmpdir(), `kimi-heic-${randomUUID()}.jpg`); + try { + const result = run( + 'sips', + ['-s', 'format', 'jpeg', '-s', 'formatOptions', '90', path, '--out', target], + { timeoutMs: SIPS_TIMEOUT_MS }, + ); + if (!result.ok) return null; + const bytes = readFileSync(target); + return bytes.length === 0 ? null : bytes; + } catch { + return null; + } finally { + try { + unlinkSync(target); + } catch { + // ignore cleanup errors + } + } +} + function readVideoPath(path: string): ClipboardVideo | null { const mimeType = videoMimeFromPath(path); if (mimeType === null) return null; @@ -215,23 +258,23 @@ function readVideoPath(path: string): ClipboardVideo | null { }; } -function readMediaPath(path: string): ClipboardMedia | null { +function readMediaPath(path: string, ctx: PathReadContext): ClipboardMedia | null { // Video files are never opened as images. const video = readVideoPath(path); if (video !== null) return video; - return readImagePath(path); + return readImagePath(path, ctx); } -function readMediaFromPaths(paths: readonly string[]): ClipboardMedia | null { +function readMediaFromPaths(paths: readonly string[], ctx: PathReadContext): ClipboardMedia | null { for (const path of paths) { - const media = readMediaPath(path); + const media = readMediaPath(path, ctx); if (media !== null) return media; } return null; } -function readMediaFromText(text: string): ClipboardMedia | null { - return readMediaFromPaths(parseClipboardPaths(text)); +function readMediaFromText(text: string, ctx: PathReadContext): ClipboardMedia | null { + return readMediaFromPaths(parseClipboardPaths(text), ctx); } function runCommand(command: string, args: string[], options?: RunCommandOptions): { stdout: Buffer; ok: boolean } { @@ -241,7 +284,7 @@ function runCommand(command: string, args: string[], options?: RunCommandOptions }); } -function readClipboardFileMediaViaWlPaste(): ClipboardMedia | null { +function readClipboardFileMediaViaWlPaste(ctx: PathReadContext): ClipboardMedia | null { const list = runCommand('wl-paste', ['--list-types'], { timeoutMs: DEFAULT_LIST_TIMEOUT_MS, }); @@ -252,7 +295,7 @@ function readClipboardFileMediaViaWlPaste(): ClipboardMedia | null { if (uriType === undefined) return null; const uris = runCommand('wl-paste', ['--type', uriType, '--no-newline']); - return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8')) : null; + return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8'), ctx) : null; } function readClipboardImageViaWlPaste(): ClipboardImage | null { @@ -269,7 +312,7 @@ function readClipboardImageViaWlPaste(): ClipboardImage | null { return { kind: 'image', bytes: data.stdout, mimeType: baseMimeType(selected) }; } -function readClipboardFileMediaViaXclip(): ClipboardMedia | null { +function readClipboardFileMediaViaXclip(ctx: PathReadContext): ClipboardMedia | null { const targets = runCommand('xclip', ['-selection', 'clipboard', '-t', 'TARGETS', '-o'], { timeoutMs: DEFAULT_LIST_TIMEOUT_MS, }); @@ -280,7 +323,7 @@ function readClipboardFileMediaViaXclip(): ClipboardMedia | null { if (uriType === undefined) return null; const uris = runCommand('xclip', ['-selection', 'clipboard', '-t', uriType, '-o']); - return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8')) : null; + return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8'), ctx) : null; } function readClipboardImageViaXclip(): ClipboardImage | null { @@ -359,6 +402,7 @@ function readClipboardFilePathsViaMacOs(run: RunCommand): string[] { async function readClipboardFileMediaViaNativeText( clip: ClipboardModule | null, + ctx: PathReadContext, ): Promise<{ media: ClipboardMedia | null; lookedFileLike: boolean }> { if (clip === null) return { media: null, lookedFileLike: false }; @@ -369,7 +413,7 @@ async function readClipboardFileMediaViaNativeText( } try { - return { media: readMediaFromText(await clip.getText()), lookedFileLike }; + return { media: readMediaFromText(await clip.getText(), ctx), lookedFileLike }; } catch (error) { if (error instanceof ClipboardMediaError) throw error; return { media: null, lookedFileLike }; @@ -408,6 +452,7 @@ export async function readClipboardMedia(options?: { const platform = options?.platform ?? process.platform; const clip = options?.clipboard ?? clipboard; const run = options?.runCommand ?? runCommand; + const ctx: PathReadContext = { run, platform }; // Termux on Android has no desktop clipboard; skip early rather than // churn through every fallback. @@ -419,7 +464,7 @@ export async function readClipboardMedia(options?: { const wsl = isWSL(env); if (wayland || wsl) { - const fileMedia = readClipboardFileMediaViaWlPaste() ?? readClipboardFileMediaViaXclip(); + const fileMedia = readClipboardFileMediaViaWlPaste(ctx) ?? readClipboardFileMediaViaXclip(ctx); if (fileMedia !== null) return fileMedia; image = readClipboardImageViaWlPaste() ?? readClipboardImageViaXclip(); } @@ -427,18 +472,18 @@ export async function readClipboardMedia(options?: { image = readClipboardImageViaPowerShell(); } if (image === null && !wayland) { - const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip); + const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip, ctx); if (nativeFileMedia.media !== null) return nativeFileMedia.media; if (nativeFileMedia.lookedFileLike) return null; image = await readClipboardImageViaNative(clip); } } else { if (platform === 'darwin') { - const fileMedia = readMediaFromPaths(readClipboardFilePathsViaMacOs(run)); + const fileMedia = readMediaFromPaths(readClipboardFilePathsViaMacOs(run), ctx); if (fileMedia !== null) return fileMedia; } - const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip); + const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip, ctx); if (nativeFileMedia.media !== null) return nativeFileMedia.media; // Finder exposes file icons/thumbnails as image data. If the clipboard diff --git a/apps/kimi-code/src/utils/image/image-mime.ts b/apps/kimi-code/src/utils/image/image-mime.ts index e7ceb97de18..d969ecb5fdc 100644 --- a/apps/kimi-code/src/utils/image/image-mime.ts +++ b/apps/kimi-code/src/utils/image/image-mime.ts @@ -25,6 +25,25 @@ export function parseImageMeta(bytes: Uint8Array): ImageMeta | null { return null; } +// ── HEIC / HEIF ───────────────────────────────────────────────────── + +const HEIC_FTYP_BRANDS = new Set(['heic', 'heix', 'hevc', 'hevx', 'heif', 'mif1', 'msf1']); + +/** + * Sniff an ISO-BMFF `ftyp` box whose major brand is one of the HEIF + * family. HEIC is not a format the model pipeline accepts, so this is + * only used to decide whether a pasted file should be converted first + * (macOS `sips`) rather than declined outright. + */ +export function isHeicImage(bytes: Uint8Array): boolean { + if (bytes.length < 12) return false; + if (bytes[4] !== 0x66 || bytes[5] !== 0x74 || bytes[6] !== 0x79 || bytes[7] !== 0x70) return false; + const brand = String.fromCodePoint(bytes[8]!, bytes[9]!, bytes[10]!, bytes[11]!) + .trim() + .toLowerCase(); + return HEIC_FTYP_BRANDS.has(brand); +} + // ── PNG ───────────────────────────────────────────────────────────── function isPng(b: Uint8Array): boolean { diff --git a/apps/kimi-code/test/utils/clipboard/clipboard-image.test.ts b/apps/kimi-code/test/utils/clipboard/clipboard-image.test.ts index 7e802db6ad1..568cdb39a1e 100644 --- a/apps/kimi-code/test/utils/clipboard/clipboard-image.test.ts +++ b/apps/kimi-code/test/utils/clipboard/clipboard-image.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -36,6 +36,42 @@ function noMacOsPaths(): { stdout: Buffer; ok: boolean } { return { stdout: Buffer.alloc(0), ok: false }; } +function heic(): Uint8Array { + const bytes = new Uint8Array(24); + bytes.set([0x00, 0x00, 0x00, 0x18], 0); + bytes.set([0x66, 0x74, 0x79, 0x70], 4); + bytes.set([0x68, 0x65, 0x69, 0x63], 8); + bytes.set([0x68, 0x65, 0x69, 0x63], 16); + return bytes; +} + +function jpeg(width: number, height: number): Uint8Array { + return new Uint8Array([ + 0xff, 0xd8, + 0xff, 0xc0, 0x00, 0x11, 0x08, + (height >> 8) & 0xff, height & 0xff, + (width >> 8) & 0xff, width & 0xff, + 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, + 0xff, 0xd9, + ]); +} + +type CommandResult = { stdout: Buffer; ok: boolean }; + +function macCommands(options: { + readonly paths: string; + readonly sips?: (args: string[]) => CommandResult; +}): { run: (command: string, args: string[]) => CommandResult; calls: string[][] } { + const calls: string[][] = []; + const run = (command: string, args: string[]): CommandResult => { + calls.push([command, ...args]); + if (command === 'osascript') return { stdout: Buffer.from(options.paths), ok: true }; + if (command === 'sips' && options.sips !== undefined) return options.sips(args); + return { stdout: Buffer.alloc(0), ok: false }; + }; + return { run, calls }; +} + describe('readClipboardMedia', () => { it('reads a copied image file from its real path instead of the Finder preview icon', async () => { const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-')); @@ -135,6 +171,85 @@ describe('readClipboardMedia', () => { expect(getImageBinary).not.toHaveBeenCalled(); }); + it('converts a copied HEIC file through sips on macOS and pastes the JPEG', async () => { + const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-')); + try { + const heicPath = join(dir, 'IMG_0001.HEIC'); + writeFileSync(heicPath, heic()); + const converted = jpeg(6, 4); + const commands = macCommands({ + paths: `${heicPath}\n`, + sips: (args) => { + writeFileSync(args[args.indexOf('--out') + 1]!, converted); + return { stdout: Buffer.alloc(0), ok: true }; + }, + }); + const clip = fakeClipboard({ availableFormats: vi.fn(() => ['public.file-url']) }); + + const media = await readClipboardMedia({ + platform: 'darwin', + clipboard: clip, + runCommand: commands.run, + }); + + expect(media).toEqual({ kind: 'image', bytes: converted, mimeType: 'image/jpeg' }); + const sips = commands.calls.find((call) => call[0] === 'sips'); + expect(sips).toBeDefined(); + expect(sips!.slice(1, 7)).toEqual(['-s', 'format', 'jpeg', '-s', 'formatOptions', '90']); + expect(sips![7]).toBe(heicPath); + expect(sips![8]).toBe('--out'); + expect(sips![9]!.endsWith('.jpg')).toBe(true); + expect(existsSync(sips![9]!)).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('declines the paste when sips cannot convert the copied HEIC', async () => { + const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-')); + try { + const heicPath = join(dir, 'broken.heic'); + writeFileSync(heicPath, heic()); + const commands = macCommands({ + paths: `${heicPath}\n`, + sips: () => ({ stdout: Buffer.alloc(0), ok: false }), + }); + const clip = fakeClipboard({ availableFormats: vi.fn(() => ['public.file-url']) }); + + const media = await readClipboardMedia({ + platform: 'darwin', + clipboard: clip, + runCommand: commands.run, + }); + + expect(media).toBeNull(); + expect(commands.calls.some((call) => call[0] === 'sips')).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('does not try to convert a copied HEIC file off macOS', async () => { + const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-')); + try { + const heicPath = join(dir, 'IMG_0002.heic'); + writeFileSync(heicPath, heic()); + const runCommand = vi.fn(() => ({ stdout: Buffer.alloc(0), ok: false })); + const clip = fakeClipboard({ + availableFormats: vi.fn(() => ['text/uri-list']), + hasText: vi.fn(() => true), + getText: vi.fn(async () => pathToFileURL(heicPath).toString()), + }); + + const media = await readClipboardMedia({ platform: 'win32', clipboard: clip, runCommand }); + + expect(media).toBeNull(); + expect(runCommand).not.toHaveBeenCalled(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('rejects pasted videos larger than 100 MB', async () => { const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-')); try { diff --git a/apps/kimi-code/test/utils/image/image-mime.test.ts b/apps/kimi-code/test/utils/image/image-mime.test.ts index bcc426bdf47..7a5ad46a197 100644 --- a/apps/kimi-code/test/utils/image/image-mime.test.ts +++ b/apps/kimi-code/test/utils/image/image-mime.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; -import { parseImageMeta } from '#/utils/image/image-mime'; +import { isHeicImage, parseImageMeta } from '#/utils/image/image-mime'; function png(width: number, height: number): Uint8Array { // 8-byte PNG signature + IHDR length (4) + 'IHDR' + width (4 BE) + height (4 BE) + ... @@ -96,3 +96,31 @@ describe('parseImageMeta', () => { expect(parseImageMeta(full.slice(0, 20))).toBeNull(); }); }); + +function ftyp(brand: string): Uint8Array { + const bytes = new Uint8Array(24); + bytes.set([0x00, 0x00, 0x00, 0x18], 0); + bytes.set([0x66, 0x74, 0x79, 0x70], 4); + bytes.set(Array.from(brand.padEnd(4, ' '), (char) => char.codePointAt(0)!), 8); + return bytes; +} + +describe('isHeicImage', () => { + it.each(['heic', 'heix', 'hevc', 'hevx', 'heif', 'mif1', 'msf1'])( + 'recognizes the %s ftyp brand', + (brand) => { + expect(isHeicImage(ftyp(brand))).toBe(true); + }, + ); + + it('rejects other ftyp brands and non-ISO-BMFF images', () => { + expect(isHeicImage(ftyp('avif'))).toBe(false); + expect(isHeicImage(ftyp('isom'))).toBe(false); + expect(isHeicImage(png(1, 1))).toBe(false); + expect(isHeicImage(new Uint8Array([0x00, 0x00, 0x00, 0x18, 0x66, 0x74]))).toBe(false); + }); + + it('is not a format parseImageMeta accepts', () => { + expect(parseImageMeta(ftyp('heic'))).toBeNull(); + }); +}); diff --git a/packages/agent-core-v2/src/agent/media/heic-transcode.ts b/packages/agent-core-v2/src/agent/media/heic-transcode.ts new file mode 100644 index 00000000000..802558dceac --- /dev/null +++ b/packages/agent-core-v2/src/agent/media/heic-transcode.ts @@ -0,0 +1,130 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { runCommand } from '#/app/capability/host'; +import type { ImageTranscodeEvent } from '#/app/telemetry/events'; +import type { ITelemetryService } from '#/app/telemetry/telemetry'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; + +import { normalizeImageMime } from './image-format-policy'; + +const HEIC_MIMES: ReadonlySet = new Set(['image/heic', 'image/heif']); + +const TRANSCODED_MIME = 'image/jpeg'; + +const SIPS_JPEG_QUALITY = '90'; + +const SIPS_TIMEOUT_MS = 15_000; + +export interface HeicTranscodeDeps { + readonly osKind: string; + readonly process: IHostProcessService | undefined; + readonly telemetry?: ITelemetryService; + readonly telemetrySource?: string; +} + +export type HeicTranscodeInput = + | { readonly path: string } + | { readonly bytes: Uint8Array }; + +export interface TranscodedImage { + readonly data: Buffer; + readonly mimeType: string; +} + +export type ImageTranscoder = ( + bytes: Uint8Array, + mimeType: string, +) => Promise; + +export function isHeicMime(mimeType: string): boolean { + return HEIC_MIMES.has(normalizeImageMime(mimeType)); +} + +export function canTranscodeHeic(deps: Pick): boolean { + return deps.osKind === 'macOS' && deps.process !== undefined; +} + +export async function transcodeHeicToJpeg( + input: HeicTranscodeInput, + mimeType: string, + deps: HeicTranscodeDeps, +): Promise { + if (!isHeicMime(mimeType) || !canTranscodeHeic(deps)) return null; + const startedAt = Date.now(); + const originalBytes = 'bytes' in input ? input.bytes.length : undefined; + const finish = ( + outcome: ImageTranscodeEvent['outcome'], + result: TranscodedImage | null, + ): TranscodedImage | null => { + reportTranscodeEvent(deps.telemetry, deps.telemetrySource, { + outcome, + startedAt, + inputMime: normalizeImageMime(mimeType), + originalBytes, + finalBytes: result?.data.length, + }); + return result; + }; + + let scratchDir: string | undefined; + try { + scratchDir = await mkdtemp(join(tmpdir(), 'kimi-heic-')); + let source: string; + if ('path' in input) { + source = input.path; + } else { + source = join(scratchDir, 'source.heic'); + await writeFile(source, input.bytes); + } + const target = join(scratchDir, 'converted.jpg'); + const result = await runCommand( + deps.process!, + 'sips', + ['-s', 'format', 'jpeg', '-s', 'formatOptions', SIPS_JPEG_QUALITY, source, '--out', target], + { timeout: SIPS_TIMEOUT_MS }, + ); + if (result.code !== 0) return finish('command_failed', null); + const data = await readFile(target).catch(() => Buffer.alloc(0)); + if (data.length === 0) return finish('empty_output', null); + return finish('converted', { data, mimeType: TRANSCODED_MIME }); + } catch { + return finish('error', null); + } finally { + if (scratchDir !== undefined) { + await rm(scratchDir, { recursive: true, force: true }).catch(() => undefined); + } + } +} + +export function createImageTranscoder(deps: HeicTranscodeDeps): ImageTranscoder { + return (bytes, mimeType) => transcodeHeicToJpeg({ bytes }, mimeType, deps); +} + +function reportTranscodeEvent( + telemetry: ITelemetryService | undefined, + source: string | undefined, + input: { + readonly outcome: ImageTranscodeEvent['outcome']; + readonly startedAt: number; + readonly inputMime: string; + readonly originalBytes: number | undefined; + readonly finalBytes: number | undefined; + }, +): void { + if (telemetry === undefined || source === undefined) return; + try { + const event: ImageTranscodeEvent = { + source, + outcome: input.outcome, + input_mime: input.inputMime, + output_mime: TRANSCODED_MIME, + original_bytes: input.originalBytes, + final_bytes: input.finalBytes, + duration_ms: Date.now() - input.startedAt, + }; + telemetry.track2('image_transcode', event); + } catch { + } +} diff --git a/packages/agent-core-v2/src/agent/media/image-originals.ts b/packages/agent-core-v2/src/agent/media/image-originals.ts index 534e4790543..cfa6201fb6a 100644 --- a/packages/agent-core-v2/src/agent/media/image-originals.ts +++ b/packages/agent-core-v2/src/agent/media/image-originals.ts @@ -13,6 +13,8 @@ const MIME_EXTENSION: Readonly> = { 'image/webp': 'webp', 'image/bmp': 'bmp', 'image/tiff': 'tif', + 'image/heic': 'heic', + 'image/heif': 'heif', }; export interface PersistOriginalImageOptions { diff --git a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts index ff51e8c17eb..82d00cd2dca 100644 --- a/packages/agent-core-v2/src/agent/media/mediaResolverService.ts +++ b/packages/agent-core-v2/src/agent/media/mediaResolverService.ts @@ -9,9 +9,13 @@ import { ITelemetryService } from '#/app/telemetry/telemetry'; import type { Message } from '#/llm-adapter/contract/message'; import type { ContentPart } from '#human/llm/message'; import type { ModelRequester } from '#/llm-adapter/model/model-requester'; +import { IHostEnvironment } from '#/os/interface/hostEnvironment'; +import { IHostProcessService } from '#/os/interface/hostProcess'; import { IBlobStore } from '#/persistence/interface/blobStore'; import { detectFileType, MEDIA_SNIFF_BYTES } from './file-type'; +import { transcodeHeicToJpeg } from './heic-transcode'; +import { compressImageForModel } from './image-compress'; import { isModelAcceptedImageMime, normalizeImageMime } from './image-format-policy'; import { buildMediaPathTag, @@ -55,6 +59,8 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { @ITelemetryService private readonly telemetry: ITelemetryService, @IAgentStateService private readonly states: IAgentStateService, @ISessionMediaStore private readonly mediaStore: ISessionMediaStore, + @IHostEnvironment private readonly hostEnv: IHostEnvironment, + @IHostProcessService private readonly hostProcess: IHostProcessService, ) { this.states.contributeState(mediaResolvedKey); } @@ -147,7 +153,11 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { source.bytes.subarray(0, MEDIA_SNIFF_BYTES), 'media', ); - if (fileType.kind !== 'image' || !isModelAcceptedImageMime(fileType.mimeType)) { + const image = + fileType.kind === 'image' + ? await this.modelImage(source.bytes, fileType.mimeType, signal) + : undefined; + if (image === undefined) { this.telemetry.track2('media_resolve_fallback', { kind: 'image', reason: 'invalid', @@ -159,15 +169,38 @@ export class AgentMediaResolverService implements IAgentMediaResolverService { const part: ContentPart = { type: 'image_url', imageUrl: { - url: `data:${normalizeImageMime(fileType.mimeType)};base64,${source.bytes.toString('base64')}`, + url: `data:${image.mimeType};base64,${image.bytes.toString('base64')}`, }, }; - if (source.bytes.length <= IMAGE_MEMO_MAX_BYTES) { - this.memoizeImage(cacheKey, part, source.bytes.length); + if (image.bytes.length <= IMAGE_MEMO_MAX_BYTES) { + this.memoizeImage(cacheKey, part, image.bytes.length); } return part; } + private async modelImage( + bytes: Buffer, + mimeType: string, + signal: AbortSignal | undefined, + ): Promise<{ readonly bytes: Buffer; readonly mimeType: string } | undefined> { + if (isModelAcceptedImageMime(mimeType)) { + return { bytes, mimeType: normalizeImageMime(mimeType) }; + } + const transcoded = await transcodeHeicToJpeg({ bytes }, mimeType, { + osKind: this.hostEnv.osKind, + process: this.hostProcess, + telemetry: this.telemetry, + telemetrySource: 'media_resolve', + }); + signal?.throwIfAborted(); + if (transcoded === null) return undefined; + const compressed = await compressImageForModel(transcoded.data, transcoded.mimeType, { + telemetry: this.telemetry, + telemetrySource: 'media_resolve', + }); + return { bytes: Buffer.from(compressed.data), mimeType: normalizeImageMime(compressed.mimeType) }; + } + private memoedImage(cacheKey: string): ContentPart | undefined { const entry = this.imageMemo.get(cacheKey); if (entry === undefined) return undefined; diff --git a/packages/agent-core-v2/src/agent/tools/read-media-file/readMediaFileTool.ts b/packages/agent-core-v2/src/agent/tools/read-media-file/readMediaFileTool.ts index 82c3bfcb69e..7c42459b177 100644 --- a/packages/agent-core-v2/src/agent/tools/read-media-file/readMediaFileTool.ts +++ b/packages/agent-core-v2/src/agent/tools/read-media-file/readMediaFileTool.ts @@ -5,6 +5,7 @@ import { inlineVideoPart, isVideoUploadAuthError } from '#/agent/media/videoUplo import type { ITelemetryService } from '#/app/telemetry/telemetry'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; import { RuntimeWorkspaceView } from '#/runtime/runtimeWorkspaceView'; import type { HostEnvironmentInfo } from '#/os/interface/hostEnvironment'; import { inspectAgentRuntime, type IAgentRuntimeService } from '#/agent/runtimeBinding/agentRuntime'; @@ -34,6 +35,7 @@ import { buildImageConversionGuidance, isModelAcceptedImageMime, } from '#/agent/media/image-format-policy'; +import { canTranscodeHeic, isHeicMime, transcodeHeicToJpeg } from '#/agent/media/heic-transcode'; import { toInputJsonSchema } from '#/tool/input-schema'; import { literalRulePattern, matchesPathRuleSubject } from '#/tool/rule-match'; import { renderPrompt } from '#/_base/utils/render-prompt'; @@ -85,12 +87,18 @@ function buildMediaNote(input: { readonly byteSize: number; readonly dimensions: { readonly width: number; readonly height: number } | null; readonly delivery?: ImageDelivery; + readonly transcodedTo?: string; }): string { const parts: string[] = [ `Read ${input.kind} file.`, `Mime type: ${input.mimeType}.`, `Size: ${String(input.byteSize)} bytes.`, ]; + if (input.transcodedTo !== undefined) { + parts.push( + `The image was converted from ${input.mimeType} to ${input.transcodedTo} before delivery.`, + ); + } if (input.kind === 'image' && input.dimensions) { parts.push( `Original dimensions: ${String(input.dimensions.width)}x${String(input.dimensions.height)} pixels.`, @@ -243,7 +251,7 @@ export class ReadMediaFileTool implements AgentTool { if (lease.runtime.identity.generation !== inspected.identity.generation) { return { isError: true, output: 'Runtime changed before execution. Retry the tool call.' }; } - return await this.execution(args, path, lease.runtime.fs!, env); + return await this.execution(args, path, lease.runtime.fs!, env, lease.runtime.process); } finally { lease.dispose(); } @@ -256,6 +264,7 @@ export class ReadMediaFileTool implements AgentTool { safePath: string, fs: IHostFileSystem, env: HostEnvironmentInfo, + process: IHostProcessService | undefined, ): Promise { if (!args.path) { return { isError: true, output: 'File path cannot be empty.' }; @@ -288,7 +297,15 @@ export class ReadMediaFileTool implements AgentTool { 'Tell the user to use a model with image input capability.', }; } - if (fileType.kind === 'image' && !isModelAcceptedImageMime(fileType.mimeType)) { + const heicTranscodable = + fileType.kind === 'image' && + isHeicMime(fileType.mimeType) && + canTranscodeHeic({ osKind: env.osKind, process }); + if ( + fileType.kind === 'image' && + !isModelAcceptedImageMime(fileType.mimeType) && + !heicTranscodable + ) { return { isError: true, output: buildImageConversionGuidance(args.path, fileType.mimeType, env.osKind), @@ -366,13 +383,28 @@ export class ReadMediaFileTool implements AgentTool { }; } - const data = Buffer.from(await fs.readBytes(safePath)); + const transcoded = heicTranscodable + ? await transcodeHeicToJpeg({ path: safePath }, fileType.mimeType, { + osKind: env.osKind, + process, + telemetry: this.telemetry, + telemetrySource: 'read_media', + }) + : null; + if (heicTranscodable && transcoded === null) { + return { + isError: true, + output: buildImageConversionGuidance(args.path, fileType.mimeType, env.osKind), + }; + } + const data = transcoded === null ? Buffer.from(await fs.readBytes(safePath)) : transcoded.data; + const imageMime = transcoded === null ? fileType.mimeType : transcoded.mimeType; let dimensions = fileType.kind === 'image' ? sniffImageDimensions(data) : null; let mediaPart: ContentPart; let delivery: ImageDelivery | undefined; if (fileType.kind === 'image') { if (args.region !== undefined) { - const outcome = await cropImageForModel(data, fileType.mimeType, args.region, { + const outcome = await cropImageForModel(data, imageMime, args.region, { skipResize: args.full_resolution === true, telemetry: this.telemetry, telemetrySource: 'read_media', @@ -405,18 +437,18 @@ export class ReadMediaFileTool implements AgentTool { const base64 = data.toString('base64'); mediaPart = { type: 'image_url', - imageUrl: { url: `data:${fileType.mimeType};base64,${base64}` }, + imageUrl: { url: `data:${imageMime};base64,${base64}` }, }; delivery = { kind: 'full', width: dimensions?.width ?? 0, height: dimensions?.height ?? 0, byteLength: data.length, - mimeType: fileType.mimeType, + mimeType: imageMime, }; } else { const { readByteBudget, maxEdge } = imageDeliveryLimits; - const compressed = await compressImageForModel(data, fileType.mimeType, { + const compressed = await compressImageForModel(data, imageMime, { byteBudget: readByteBudget, maxEdge, telemetry: this.telemetry, @@ -465,6 +497,7 @@ export class ReadMediaFileTool implements AgentTool { byteSize: stat.size, dimensions, delivery, + transcodedTo: transcoded?.mimeType, }); const output: ContentPart[] = [ diff --git a/packages/agent-core-v2/src/app/telemetry/events.ts b/packages/agent-core-v2/src/app/telemetry/events.ts index 465cce28e37..754df2bb86d 100644 --- a/packages/agent-core-v2/src/app/telemetry/events.ts +++ b/packages/agent-core-v2/src/app/telemetry/events.ts @@ -435,6 +435,16 @@ export interface ImageCompressEvent { duration_ms: number; } +export interface ImageTranscodeEvent { + source: string; + outcome: 'converted' | 'command_failed' | 'empty_output' | 'error'; + input_mime: string; + output_mime: string; + original_bytes?: number; + final_bytes?: number; + duration_ms: number; +} + export interface ImageCropEvent { source: string; ok: boolean; @@ -1086,6 +1096,19 @@ export const telemetryEventDefinitions = { duration_ms: 'Compression wall-clock time in milliseconds', }, }), + image_transcode: defineTelemetryEvent({ + owner: 'kimi-code', + comment: 'An image in a format the provider rejects is converted with a host tool before being sent to the model.', + properties: { + source: 'Where the image came from', + outcome: 'Conversion outcome', + input_mime: 'Input MIME type', + output_mime: 'Output MIME type', + original_bytes: 'Input size in bytes when known', + final_bytes: 'Output size in bytes when converted', + duration_ms: 'Conversion wall-clock time in milliseconds', + }, + }), image_crop: defineTelemetryEvent({ owner: 'kimi-code', comment: 'An image is cropped to a region before being sent to the model.', diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index 45bfedaea4c..1e6b2ff1779 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -615,6 +615,16 @@ export { persistOriginalImage, sessionMediaOriginalsDir, } from '#/agent/media/image-originals'; +export { + canTranscodeHeic, + createImageTranscoder, + isHeicMime, + transcodeHeicToJpeg, + type HeicTranscodeDeps, + type HeicTranscodeInput, + type ImageTranscoder, + type TranscodedImage, +} from '#/agent/media/heic-transcode'; export * from '#/app/edit/fileEdit'; export * from '#/app/edit/fileEditService'; export * from '#/app/edit/editService'; diff --git a/packages/agent-core-v2/test/agent/media/fakeSips.ts b/packages/agent-core-v2/test/agent/media/fakeSips.ts new file mode 100644 index 00000000000..e08a9b73140 --- /dev/null +++ b/packages/agent-core-v2/test/agent/media/fakeSips.ts @@ -0,0 +1,73 @@ +import { writeFileSync } from 'node:fs'; +import { Readable, Writable } from 'node:stream'; + +import type { IHostProcess, IHostProcessService } from '#/os/interface/hostProcess'; + +export function tinyJpeg(width = 4, height = 3): Buffer { + return Buffer.from([ + 0xff, 0xd8, + 0xff, 0xc0, 0x00, 0x11, 0x08, + (height >> 8) & 0xff, height & 0xff, + (width >> 8) & 0xff, width & 0xff, + 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, + 0xff, 0xd9, + ]); +} + +export function heicBytes(brand = 'heic'): Buffer { + const buf = Buffer.alloc(24); + buf.writeUInt32BE(24, 0); + buf.write('ftyp', 4, 'latin1'); + buf.write(brand, 8, 'latin1'); + buf.write(brand, 16, 'latin1'); + return buf; +} + +export interface FakeSipsCall { + readonly command: string; + readonly args: readonly string[]; +} + +export interface FakeSipsOptions { + readonly output?: Uint8Array | null; + readonly exitCode?: number; + readonly spawnError?: Error; +} + +export interface FakeSips { + readonly service: IHostProcessService; + readonly calls: FakeSipsCall[]; +} + +export function fakeSips(options: FakeSipsOptions = {}): FakeSips { + const calls: FakeSipsCall[] = []; + const service: IHostProcessService = { + _serviceBrand: undefined, + spawn: async (command, args = []) => { + calls.push({ command, args: [...args] }); + if (options.spawnError !== undefined) throw options.spawnError; + const exitCode = options.exitCode ?? 0; + const outIndex = args.indexOf('--out'); + const target = outIndex === -1 ? undefined : args[outIndex + 1]; + if (exitCode === 0 && target !== undefined && options.output !== null) { + writeFileSync(target, options.output ?? tinyJpeg()); + } + return fakeProcess(exitCode); + }, + }; + return { service, calls }; +} + +function fakeProcess(exitCode: number): IHostProcess { + return { + _serviceBrand: undefined, + pid: 4242, + exitCode, + stdin: new Writable({ write: (_chunk, _encoding, callback) => callback() }), + stdout: Readable.from([]), + stderr: Readable.from([]), + wait: async () => exitCode, + kill: async () => {}, + dispose: () => {}, + }; +} diff --git a/packages/agent-core-v2/test/agent/media/heic-transcode.test.ts b/packages/agent-core-v2/test/agent/media/heic-transcode.test.ts new file mode 100644 index 00000000000..5e705a844cf --- /dev/null +++ b/packages/agent-core-v2/test/agent/media/heic-transcode.test.ts @@ -0,0 +1,210 @@ +import { existsSync } from 'node:fs'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; + +import { Jimp } from 'jimp'; +import { describe, expect, it } from 'vitest'; + +import { runCommand } from '#/app/capability/host'; +import type { ITelemetryService, TelemetryProperties } from '#/app/telemetry/telemetry'; +import { sniffImageDimensions } from '#/agent/media/file-type'; +import { + canTranscodeHeic, + createImageTranscoder, + isHeicMime, + transcodeHeicToJpeg, +} from '#/agent/media/heic-transcode'; +import { HostProcessService } from '#/os/backends/node-local/hostProcessService'; + +import { fakeSips, heicBytes, tinyJpeg } from './fakeSips'; + +const MAC = 'macOS'; + +interface Recorded { + readonly event: string; + readonly properties: Readonly> | undefined; +} + +function recordingTelemetry(records: Recorded[]): ITelemetryService { + const telemetry: ITelemetryService = { + _serviceBrand: undefined, + track2(event, properties) { + records.push({ event, properties: properties as TelemetryProperties }); + }, + withContext: () => telemetry, + setContext: () => {}, + getContext: () => ({}), + addAppender: () => ({ dispose: () => {} }), + removeAppender: () => {}, + setEnabled: () => {}, + flush: async () => {}, + shutdown: async () => {}, + }; + return telemetry; +} + +describe('isHeicMime', () => { + it('recognizes HEIC and HEIF, ignoring case and parameters', () => { + expect(isHeicMime('image/heic')).toBe(true); + expect(isHeicMime('image/heif')).toBe(true); + expect(isHeicMime('IMAGE/HEIC; charset=binary')).toBe(true); + expect(isHeicMime('image/avif')).toBe(false); + expect(isHeicMime('image/jpeg')).toBe(false); + }); +}); + +describe('canTranscodeHeic', () => { + it('offers conversion only on macOS with a process service', () => { + const sips = fakeSips(); + expect(canTranscodeHeic({ osKind: MAC, process: sips.service })).toBe(true); + expect(canTranscodeHeic({ osKind: 'Linux', process: sips.service })).toBe(false); + expect(canTranscodeHeic({ osKind: 'Windows', process: sips.service })).toBe(false); + expect(canTranscodeHeic({ osKind: MAC, process: undefined })).toBe(false); + }); +}); + +describe('transcodeHeicToJpeg', () => { + it('returns null without spawning anything on hosts other than macOS', async () => { + const sips = fakeSips(); + + const result = await transcodeHeicToJpeg({ bytes: heicBytes() }, 'image/heic', { + osKind: 'Linux', + process: sips.service, + }); + + expect(result).toBeNull(); + expect(sips.calls).toEqual([]); + }); + + it('returns null without spawning for images that are not HEIC', async () => { + const sips = fakeSips(); + + const result = await transcodeHeicToJpeg({ bytes: Buffer.from('BM') }, 'image/bmp', { + osKind: MAC, + process: sips.service, + }); + + expect(result).toBeNull(); + expect(sips.calls).toEqual([]); + }); + + it('converts HEIC bytes through sips into JPEG and removes its scratch directory', async () => { + const sips = fakeSips({ output: tinyJpeg(6, 4) }); + + const result = await transcodeHeicToJpeg({ bytes: heicBytes() }, 'image/heic', { + osKind: MAC, + process: sips.service, + }); + + expect(result).toEqual({ data: tinyJpeg(6, 4), mimeType: 'image/jpeg' }); + expect(sips.calls).toHaveLength(1); + const call = sips.calls[0]!; + expect(call.command).toBe('sips'); + const outIndex = call.args.indexOf('--out'); + expect(call.args.slice(0, outIndex - 1)).toEqual([ + '-s', 'format', 'jpeg', '-s', 'formatOptions', '90', + ]); + const source = call.args[outIndex - 1]!; + const target = call.args[outIndex + 1]!; + expect(source.endsWith('.heic')).toBe(true); + expect(target.endsWith('.jpg')).toBe(true); + expect(dirname(source)).toBe(dirname(target)); + expect(existsSync(dirname(target))).toBe(false); + }); + + it('hands a path input to sips directly instead of copying the file', async () => { + const sips = fakeSips(); + + const result = await transcodeHeicToJpeg({ path: '/photos/IMG_0001.HEIC' }, 'image/heif', { + osKind: MAC, + process: sips.service, + }); + + expect(result?.mimeType).toBe('image/jpeg'); + const call = sips.calls[0]!; + const outIndex = call.args.indexOf('--out'); + expect(call.args[outIndex - 1]).toBe('/photos/IMG_0001.HEIC'); + expect(existsSync(dirname(call.args[outIndex + 1]!))).toBe(false); + }); + + it.each([ + { name: 'sips exits non-zero', options: { exitCode: 1 } }, + { name: 'sips writes no output file', options: { output: null } }, + { name: 'sips cannot be spawned', options: { spawnError: new Error('ENOENT') } }, + ])('returns null and cleans up when $name', async ({ options }) => { + const sips = fakeSips(options); + + const result = await transcodeHeicToJpeg({ bytes: heicBytes() }, 'image/heic', { + osKind: MAC, + process: sips.service, + }); + + expect(result).toBeNull(); + const call = sips.calls[0]!; + const outIndex = call.args.indexOf('--out'); + expect(existsSync(dirname(call.args[outIndex + 1]!))).toBe(false); + }); + + it('reports the conversion outcome to telemetry', async () => { + const records: Recorded[] = []; + const telemetry = recordingTelemetry(records); + + await transcodeHeicToJpeg({ bytes: heicBytes() }, 'image/heic', { + osKind: MAC, + process: fakeSips({ output: tinyJpeg() }).service, + telemetry, + telemetrySource: 'read_media', + }); + await transcodeHeicToJpeg({ bytes: heicBytes() }, 'image/heic', { + osKind: MAC, + process: fakeSips({ exitCode: 2 }).service, + telemetry, + telemetrySource: 'read_media', + }); + + expect(records.map((record) => record.event)).toEqual(['image_transcode', 'image_transcode']); + expect(records[0]!.properties).toMatchObject({ + source: 'read_media', + outcome: 'converted', + input_mime: 'image/heic', + original_bytes: heicBytes().length, + final_bytes: tinyJpeg().length, + }); + expect(records[1]!.properties).toMatchObject({ outcome: 'command_failed' }); + }); + + it('createImageTranscoder binds the host so callers only pass bytes', async () => { + const sips = fakeSips({ output: tinyJpeg(2, 2) }); + const transcode = createImageTranscoder({ osKind: MAC, process: sips.service }); + + expect(await transcode(heicBytes(), 'image/heic')).toEqual({ + data: tinyJpeg(2, 2), + mimeType: 'image/jpeg', + }); + expect(await transcode(Buffer.from('BM'), 'image/bmp')).toBeNull(); + }); + + describe.skipIf(process.platform !== 'darwin')('with the real macOS sips', () => { + it('converts a HEIC written by sips itself back into a decodable JPEG', async () => { + const dir = await mkdtemp(join(tmpdir(), 'heic-real-')); + try { + const pngPath = join(dir, 'source.png'); + const heicPath = join(dir, 'source.heic'); + await new Jimp({ width: 40, height: 24, color: 0x3366ccff }).write(pngPath as `${string}.png`); + const process = new HostProcessService(); + const encoded = await runCommand(process, 'sips', ['-s', 'format', 'heic', pngPath, '--out', heicPath]); + expect(encoded.code).toBe(0); + const bytes = await readFile(heicPath); + + const result = await transcodeHeicToJpeg({ bytes }, 'image/heic', { osKind: MAC, process }); + + expect(result?.mimeType).toBe('image/jpeg'); + expect([...result!.data.subarray(0, 3)]).toEqual([0xff, 0xd8, 0xff]); + expect(sniffImageDimensions(result!.data)).toMatchObject({ width: 40, height: 24 }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + }); +}); diff --git a/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts b/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts index 602dc029dd8..6ecd283ade7 100644 --- a/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts +++ b/packages/agent-core-v2/test/agent/media/mediaResolver.test.ts @@ -26,9 +26,13 @@ import type { Message } from '#/llm-adapter/contract/message'; import type { ContentPart, VideoURLPart } from '#human/llm/message'; import type { ModelRequester } from '#/llm-adapter/model/model-requester'; import type { Protocol } from '#/llm-adapter/protocol/protocol'; +import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; +import { IHostEnvironment as IHostEnvironmentId } from '#/os/interface/hostEnvironment'; +import { IHostProcessService } from '#/os/interface/hostProcess'; import { IBlobStore } from '#/persistence/interface/blobStore'; import { registerStateServices } from '../../state/stubs'; +import { fakeSips, heicBytes, tinyJpeg } from './fakeSips'; const FILE_ID = 'file_abc'; const VIDEO_BYTES = Buffer.from('tiny fake mp4 bytes'); @@ -127,6 +131,20 @@ function blobStore(): IBlobStore { const telemetry = { track2: () => {} } as unknown as ITelemetryService; +function hostEnv(osKind = 'Linux'): IHostEnvironment { + return { + _serviceBrand: undefined, + osKind, + osArch: 'arm64', + osVersion: 'test', + shellName: 'sh', + shellPath: '/bin/sh', + pathClass: 'posix', + homeDir: '/home/test', + ready: Promise.resolve(), + }; +} + function stubMediaStore(sessionDir = '/nonexistent-session'): ISessionMediaStore { return { _serviceBrand: undefined, @@ -205,6 +223,10 @@ function resolver( files: Map, sessionDir?: string, mediaStore: ISessionMediaStore = stubMediaStore(sessionDir), + host: { readonly osKind: string; readonly process: IHostProcessService } = { + osKind: 'Linux', + process: fakeSips().service, + }, ): IAgentMediaResolverService { const ix = createServices(disposables, { base: [registerStateServices], @@ -213,6 +235,8 @@ function resolver( reg.defineInstance(IBlobStore, blobStore()); reg.defineInstance(ITelemetryService, telemetry); reg.defineInstance(ISessionMediaStore, mediaStore); + reg.defineInstance(IHostEnvironmentId, hostEnv(host.osKind)); + reg.defineInstance(IHostProcessService, host.process); reg.define(IAgentMediaResolverService, AgentMediaResolverService); }, }); @@ -263,13 +287,13 @@ describe('AgentMediaResolverService video strategy', () => { const message = videoMessage(buildKimiFileUrl(FILE_ID)); const upload1 = vi.fn(async (): Promise => msPart('prov-1')); - await new AgentMediaResolverService(fileService(files), blobs, telemetry, new AgentStateService(), stubMediaStore()).resolve( + await new AgentMediaResolverService(fileService(files), blobs, telemetry, new AgentStateService(), stubMediaStore(), hostEnv(), fakeSips().service).resolve( [message], requester({ uploadVideo: upload1 }), ); const upload2 = vi.fn(async (): Promise => msPart('prov-2')); - const out = await new AgentMediaResolverService(fileService(files), blobs, telemetry, new AgentStateService(), stubMediaStore()).resolve( + const out = await new AgentMediaResolverService(fileService(files), blobs, telemetry, new AgentStateService(), stubMediaStore(), hostEnv(), fakeSips().service).resolve( [message], requester({ uploadVideo: upload2 }), ); @@ -478,6 +502,8 @@ describe('AgentMediaResolverService image strategy', () => { telemetry, new AgentStateService(), stubMediaStore(), + hostEnv(), + fakeSips().service, ); await expect( @@ -514,6 +540,12 @@ describe('AgentMediaResolverService image strategy', () => { fileId: FILE_ID, imageIn: true, }, + { + name: 'the bytes sniff as HEIC on a host without a converter', + files: new Map([[FILE_ID, { name: 'pic.heic', bytes: heicBytes() }]]), + fileId: FILE_ID, + imageIn: true, + }, ])('degrades when $name', async ({ files, fileId, imageIn }) => { const canonical = fileId === FILE_ID ? await plantCanonical(FILE_ID, '.png', PNG_BYTES) : undefined; @@ -560,6 +592,8 @@ describe('AgentMediaResolverService image strategy', () => { telemetry, new AgentStateService(), stubMediaStore(), + hostEnv(), + fakeSips().service, ); const message = imageMessage(buildKimiFileUrl(FILE_ID)); const expected = { type: 'image_url', imageUrl: { url: PNG_DATA_URL } }; @@ -586,6 +620,8 @@ describe('AgentMediaResolverService image strategy', () => { telemetry, new AgentStateService(), stubMediaStore(), + hostEnv(), + fakeSips().service, ); const message = imageMessage(buildKimiFileUrl(FILE_ID)); @@ -608,6 +644,8 @@ describe('AgentMediaResolverService image strategy', () => { telemetry, new AgentStateService(), stubMediaStore(), + hostEnv(), + fakeSips().service, ); const req = requester({}); @@ -650,6 +688,8 @@ describe('AgentMediaResolverService image strategy', () => { telemetry, new AgentStateService(), stubMediaStore(sessionDir), + hostEnv(), + fakeSips().service, ); const message = imageMessage(buildKimiFileUrl(FILE_ID)); @@ -777,6 +817,8 @@ describe('AgentMediaResolverService scoped registration', () => { stubPair(IFileService, fileService(files)), stubPair(IBlobStore, blobStore()), stubPair(ITelemetryService, telemetry), + stubPair(IHostEnvironmentId, hostEnv()), + stubPair(IHostProcessService, fakeSips().service), ]); return host.child(LifecycleScope.Agent, 'main', [ stubPair(IAgentStateService, new AgentStateService()), @@ -796,3 +838,36 @@ describe('AgentMediaResolverService scoped registration', () => { expect(firstPart(out)).toEqual({ type: 'image_url', imageUrl: { url: PNG_DATA_URL } }); }); }); + +describe('AgentMediaResolverService HEIC conversion', () => { + const heicFiles = () => new Map([[FILE_ID, { name: 'pic.heic', bytes: heicBytes() }]]); + + it('converts a HEIC upload through the host sips on macOS and inlines the JPEG once', async () => { + const sips = fakeSips({ output: tinyJpeg(6, 4) }); + const res = resolver(heicFiles(), undefined, undefined, { osKind: 'macOS', process: sips.service }); + const message = imageMessage(buildKimiFileUrl(FILE_ID)); + + const first = await res.resolve([message], requester({})); + const second = await res.resolve([message], requester({})); + + const expected = { + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${tinyJpeg(6, 4).toString('base64')}` }, + }; + expect(firstPart(first)).toEqual(expected); + expect(firstPart(second)).toEqual(expected); + expect(sips.calls).toHaveLength(1); + }); + + it('degrades to the path tag when sips fails on macOS', async () => { + const canonical = await plantCanonical(FILE_ID, '.heic', heicBytes()); + const res = resolver(heicFiles(), sessionDir, undefined, { + osKind: 'macOS', + process: fakeSips({ exitCode: 1 }).service, + }); + + const out = await res.resolve([imageMessage(buildKimiFileUrl(FILE_ID))], requester({})); + + expect(firstPart(out)).toEqual({ type: 'text', text: `` }); + }); +}); diff --git a/packages/agent-core-v2/test/agent/media/tools/read-media.test.ts b/packages/agent-core-v2/test/agent/media/tools/read-media.test.ts index d3d9facf7e8..d20d0282929 100644 --- a/packages/agent-core-v2/test/agent/media/tools/read-media.test.ts +++ b/packages/agent-core-v2/test/agent/media/tools/read-media.test.ts @@ -13,6 +13,7 @@ import { } from '#/_base/errors/unexpectedError'; import type { IHostFileSystem } from '#/os/interface/hostFileSystem'; import type { IHostEnvironment } from '#/os/interface/hostEnvironment'; +import type { IHostProcessService } from '#/os/interface/hostProcess'; import type { IAgentRuntimeService } from '#/agent/runtimeBinding/agentRuntime'; import type { Runtime } from '#/runtime/runtime'; import type { ITelemetryService, TelemetryProperties } from '#/app/telemetry/telemetry'; @@ -45,6 +46,7 @@ import type { ISessionWorkspaceContext } from '#/session/workspaceContext/worksp import type { WorkspaceConfig } from '#/tool/path-access'; import { sniffImageDimensions } from '#/agent/media/file-type'; import { stubAgentContext } from '../../agentContext/stubs'; +import { fakeSips, tinyJpeg } from '../fakeSips'; const WORKSPACE: WorkspaceConfig = { workspaceDir: '/workspace', additionalDirs: [] }; @@ -156,10 +158,10 @@ function createTestFs(files: Record): IHostFileSystem { } as unknown as IHostFileSystem; } -function createTestEnv(): IHostEnvironment { +function createTestEnv(osKind = 'Linux'): IHostEnvironment { return { _serviceBrand: undefined, - osKind: 'Linux', + osKind, osArch: 'x86_64', osVersion: 'test', shellName: 'bash', @@ -170,14 +172,19 @@ function createTestEnv(): IHostEnvironment { }; } -function runtimeFor(fs: IHostFileSystem, env: IHostEnvironment = createTestEnv()): IAgentRuntimeService { +function runtimeFor( + fs: IHostFileSystem, + env: IHostEnvironment = createTestEnv(), + process?: IHostProcessService, +): IAgentRuntimeService { const runtime = { identity: { workspaceId: 'workspace', runtimeId: 'local', generation: 'test' }, - capabilities: new Set(['fs'] as const), + capabilities: new Set(process === undefined ? (['fs'] as const) : (['fs', 'process'] as const)), environment: env, path: posixPath, workspace: { mapRoots: (roots: { workDir: string; additionalDirs?: readonly string[] }) => roots }, fs, + process, status: 'ready', onDidChangeStatus: () => ({ dispose: () => {} }), dispose: () => {}, @@ -1122,4 +1129,70 @@ describe('createVideoUploader', () => { expect(result.output).toMatch(/sips -s format jpeg|magick/); expect(result.output).not.toContain('heif-convert'); }); + + function macTool( + files: Record, + process: IHostProcessService | undefined, + osKind = 'macOS', + ): ReadMediaFileTool { + return new ReadMediaFileTool( + runtimeFor(createTestFs(files), createTestEnv(osKind), process), + WORKSPACE, + capabilities(), + ); + } + + it('converts HEIC through the host sips on macOS and delivers the JPEG', async () => { + const sips = fakeSips({ output: tinyJpeg(6, 4) }); + + const result = await execute(macTool({ '/workspace/photo.heic': { data: heicBytes() } }, sips.service), { + path: '/workspace/photo.heic', + }); + + const parts = outputParts(result); + expect(parts[1]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${tinyJpeg(6, 4).toString('base64')}` }, + }); + const note = noteText(result); + expect(note).toContain('Mime type: image/heic.'); + expect(note).toContain(`Size: ${String(heicBytes().length)} bytes.`); + expect(note).toContain('Original dimensions: 6x4 pixels.'); + expect(note).toContain('converted from image/heic to image/jpeg'); + const call = sips.calls[0]!; + expect(call.command).toBe('sips'); + expect(call.args[call.args.indexOf('--out') - 1]).toBe('/workspace/photo.heic'); + }); + + it('crops a region of a HEIC from the converted JPEG', async () => { + const jpeg = await new Jimp({ width: 40, height: 24, color: 0x3366ccff }).getBuffer('image/jpeg'); + const sips = fakeSips({ output: jpeg }); + + const result = await execute(macTool({ '/workspace/photo.heic': { data: heicBytes() } }, sips.service), { + path: '/workspace/photo.heic', + region: { x: 0, y: 0, width: 10, height: 8 }, + }); + + const parts = outputParts(result); + const image = parts[1] as { type: 'image_url'; imageUrl: { url: string } }; + expect(image.imageUrl.url.startsWith('data:image/jpeg;base64,')).toBe(true); + const note = noteText(result); + expect(note).toContain('Showing region (x=0, y=0, width=10, height=8)'); + expect(note).toContain('Original dimensions: 40x24 pixels.'); + }); + + it.each([ + { name: 'sips fails', process: () => fakeSips({ exitCode: 1 }).service, osKind: 'macOS' }, + { name: 'the runtime has no process capability', process: () => undefined, osKind: 'macOS' }, + { name: 'the host is not macOS', process: () => fakeSips().service, osKind: 'Windows' }, + ])('falls back to the conversion guidance when $name', async ({ process, osKind }) => { + const result = await execute( + macTool({ '/workspace/photo.heic': { data: heicBytes() } }, process(), osKind), + { path: '/workspace/photo.heic' }, + ); + + expect(result.isError).toBe(true); + expect(result.output).toContain('image/heic'); + expect(result.output).toContain('Convert it to JPEG first'); + }); }); diff --git a/packages/kap-server/src/lib/promptMedia.ts b/packages/kap-server/src/lib/promptMedia.ts index b82df77dd3f..7111d353079 100644 --- a/packages/kap-server/src/lib/promptMedia.ts +++ b/packages/kap-server/src/lib/promptMedia.ts @@ -23,9 +23,11 @@ import { type ContentPart, type GetResult, type IFileService, + type ImageTranscoder, type ISessionMediaStore, type ITelemetryService, type PromptFileAttachment, + type TranscodedImage, } from '@moonshot-ai/agent-core-v2'; import { sniffMediaFromMagic } from '@moonshot-ai/agent-core-v2/agent/media/file-type'; import { @@ -131,6 +133,7 @@ export interface ResolvePromptMediaOptions { readonly resolveOriginalsDir?: () => Promise; readonly resolveAttachmentsDir?: () => Promise; readonly telemetry?: ITelemetryService; + readonly transcodeImage?: ImageTranscoder; } export interface PromptMediaPreparation { @@ -184,6 +187,32 @@ export async function resolvePromptMediaFiles( ); if (!isModelAcceptedImageMime(effectiveMime)) { const bytes = Buffer.from(part.source.data, 'base64'); + const transcoded = await options.transcodeImage?.(bytes, effectiveMime); + if (transcoded !== undefined && transcoded !== null) { + const image = await transcodedModelImage( + transcoded, + { + bytes, + mimeType: effectiveMime, + path: await persistOriginalImage(bytes, effectiveMime, { + dir: await resolveOriginalsDir(), + }), + }, + options.telemetry, + ); + content.push({ type: 'text', text: image.caption }); + content.push({ + type: 'image', + source: { + kind: 'base64', + media_type: image.mimeType, + data: image.data.toString('base64'), + }, + name: part.name, + }); + changed = true; + continue; + } const name = part.name ?? `image.${imageExtensionForMime(effectiveMime)}`; const persisted = await persistAttachmentBytes( bytes, @@ -308,6 +337,28 @@ export async function resolvePromptMediaFiles( } let mediaType = resolveEffectiveImageMime(declared, data); if (!isModelAcceptedImageMime(mediaType)) { + const transcoded = await options.transcodeImage?.(data, mediaType); + if (transcoded !== undefined && transcoded !== null) { + const image = await transcodedModelImage( + transcoded, + { bytes: data, mimeType: mediaType, path: sourcePath }, + options.telemetry, + ); + const saved = await store.save( + Readable.from(image.data), + compressedUploadName(name, image.mimeType), + { mimeType: image.mimeType }, + ); + ownedFileIds.add(saved.id); + content.push({ type: 'text', text: image.caption }); + content.push({ + type: 'image', + source: { kind: 'url', url: buildDaemonFileUrl(saved.id) }, + name: part.name ?? name, + }); + changed = true; + continue; + } content.push({ type: 'text', text: buildAttachedFileNotice(name, mediaType, data.length, sourcePath), @@ -391,6 +442,34 @@ export async function resolvePromptMediaFiles( let mediaType = file.meta.media_type; mediaType = resolveEffectiveImageMime(mediaType, data); if (!isModelAcceptedImageMime(mediaType)) { + const transcoded = await options.transcodeImage?.(data, mediaType); + if (transcoded !== undefined && transcoded !== null) { + const image = await transcodedModelImage( + transcoded, + { + bytes: data, + mimeType: mediaType, + path: await persistOriginalImage(data, mediaType, { + dir: await resolveOriginalsDir(), + }), + }, + options.telemetry, + ); + const saved = await store.save( + Readable.from(image.data), + compressedUploadName(file.meta.name, image.mimeType), + { mimeType: image.mimeType }, + ); + ownedFileIds.add(saved.id); + content.push({ type: 'text', text: image.caption }); + content.push({ + type: 'image', + source: { kind: 'url', url: buildDaemonFileUrl(saved.id) }, + name: part.name ?? file.meta.name, + }); + changed = true; + continue; + } const name = part.name ?? file.meta.name; const persisted = await persistAttachmentBytes( data, @@ -479,6 +558,36 @@ function compressedUploadName(originalName: string, mimeType: string): string { return `${base.length > 0 ? base : 'image'}.${imageExtensionForMime(mimeType)}`; } +interface ModelImage { + readonly data: Buffer; + readonly mimeType: string; + readonly caption: string; +} + +async function transcodedModelImage( + transcoded: TranscodedImage, + original: { readonly bytes: Buffer; readonly mimeType: string; readonly path: string | null }, + telemetry: ITelemetryService | undefined, +): Promise { + const compressed = await compressImageForModel(transcoded.data, transcoded.mimeType, { + telemetry, + telemetrySource: 'prompt_transcode', + }); + const data = compressed.changed ? Buffer.from(compressed.data) : transcoded.data; + const mimeType = compressed.changed ? compressed.mimeType : transcoded.mimeType; + const caption = buildImageCompressionCaption({ + original: { + width: compressed.originalWidth, + height: compressed.originalHeight, + byteLength: original.bytes.length, + mimeType: original.mimeType, + }, + final: { width: compressed.width, height: compressed.height, byteLength: data.length, mimeType }, + originalPath: original.path, + }); + return { data, mimeType, caption }; +} + const ATTACHMENT_NAME_MAX = 100; function sanitizeAttachmentName(name: string): string { diff --git a/packages/kap-server/src/routes/prompts.ts b/packages/kap-server/src/routes/prompts.ts index 50e2ab8a6c4..30fb27a2674 100644 --- a/packages/kap-server/src/routes/prompts.ts +++ b/packages/kap-server/src/routes/prompts.ts @@ -13,6 +13,9 @@ import { IEventBus, IEventService, IFileService, + IHostEnvironment, + IHostProcessService, + createImageTranscoder, ISessionMediaStore, ISessionMetadata, ISessionSkillCatalog, @@ -264,6 +267,12 @@ export function registerPromptsRoutes(app: PromptRouteHost, core: Scope): void { core.accessor.get(IBootstrapService).cacheDir, { telemetry, + transcodeImage: createImageTranscoder({ + osKind: core.accessor.get(IHostEnvironment).osKind, + process: core.accessor.get(IHostProcessService), + telemetry, + telemetrySource: 'prompt', + }), resolveOriginalsDir: async () => { const session = await resumeSessionById(core.accessor, session_id); if (session === undefined) return undefined; diff --git a/packages/kap-server/src/routes/skills.ts b/packages/kap-server/src/routes/skills.ts index c7890f2979d..79f7d81ab4d 100644 --- a/packages/kap-server/src/routes/skills.ts +++ b/packages/kap-server/src/routes/skills.ts @@ -9,6 +9,9 @@ import { IBootstrapService, IConfigService, IFileService, + IHostEnvironment, + IHostProcessService, + createImageTranscoder, IFlagService, IPluginService, ISessionContext, @@ -261,6 +264,12 @@ export function registerSkillsRoutes(app: SkillsRouteHost, core: Scope): void { { telemetry, resolveOriginalsDir: async () => sessionMediaOriginalsDir(sessionDir), + transcodeImage: createImageTranscoder({ + osKind: core.accessor.get(IHostEnvironment).osKind, + process: core.accessor.get(IHostProcessService), + telemetry, + telemetrySource: 'prompt', + }), resolveAttachmentsDir: async () => join(sessionDir, 'attachments'), }, ); diff --git a/packages/kap-server/test/prompts.test.ts b/packages/kap-server/test/prompts.test.ts index 6ab1ae06858..c34a6d9810a 100644 --- a/packages/kap-server/test/prompts.test.ts +++ b/packages/kap-server/test/prompts.test.ts @@ -1,6 +1,8 @@ +import { execFileSync } from 'node:child_process'; import { chmod, mkdir, mkdtemp, open, readFile, readdir, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; +import { Readable } from 'node:stream'; import { deflateSync } from 'node:zlib'; import { @@ -21,11 +23,13 @@ import { MAX_IMAGE_DECODE_BYTES, closeSessionById, getLiveSessionById, + type FileMeta, } from '@moonshot-ai/agent-core-v2'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { type RunningServer, startServer } from '../src/start'; import { projectPromptSnapshot, watchPromptSettlements } from '../src/routes/prompts'; +import { resolvePromptMediaFiles } from '../src/lib/promptMedia'; import { TEST_HOST_IDENTITY } from './helpers/hostIdentity'; import { authHeaders } from './helpers/auth'; @@ -1046,6 +1050,50 @@ describe('server-v2 /api/v1 prompts', () => { return uploaded.data; } + it.skipIf(process.platform !== 'darwin')( + 'converts an uploaded HEIC through the macOS sips before sending it to the model', + async () => { + const id = await createSession(home as string); + await createMainAgent(id); + const scratch = await mkdtemp(join(tmpdir(), 'kap-heic-')); + try { + const pngPath = join(scratch, 'photo.png'); + const heicPath = join(scratch, 'photo.heic'); + await writeFile(pngPath, solidPng(40, 24)); + execFileSync('sips', ['-s', 'format', 'heic', pngPath, '--out', heicPath], { stdio: 'ignore' }); + const heic = await readFile(heicPath); + const uploaded = await uploadFile(heic, 'image/heic', 'photo.heic'); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'image', source: { kind: 'file', file_id: uploaded.id } }], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as Array>; + expect(content).toHaveLength(2); + const caption = content[0] as { type: string; text: string }; + expect(caption.type).toBe('text'); + expect(caption.text).toContain('image/heic'); + expect(caption.text).toContain('image/jpeg'); + expect(caption.text).toContain('40x24'); + const image = content[1] as { type: string; source: { kind: string; file_id: string } }; + expect(image.type).toBe('image'); + expect(image.source.kind).toBe('session_media'); + const mediaDir = sessionMediaDir(server!, id); + const mediaFile = await vi.waitFor(async () => { + const names = await readdir(mediaDir); + const name = names.find((entry) => entry.startsWith(image.source.file_id)); + expect(name).toBeDefined(); + return join(mediaDir, name!); + }); + expect(/\.jpe?g$/.test(mediaFile)).toBe(true); + expect([...(await readFile(mediaFile)).subarray(0, 3)]).toEqual([0xff, 0xd8, 0xff]); + } finally { + await rm(scratch, { recursive: true, force: true }); + } + }, + ); + function attachedPathFrom(notice: string): string { const match = /bytes\): (.+) — open it with the Read tool$/.exec(notice); expect(match).not.toBeNull(); @@ -1726,3 +1774,196 @@ describe('server-v2 /api/v1 prompts', () => { expect(toolPolicy?.isToolActive('Read')).toBe(true); }); }); + +describe('resolvePromptMediaFiles HEIC conversion', () => { + function heicBytes(): Buffer { + const buf = Buffer.alloc(24); + buf.writeUInt32BE(24, 0); + buf.write('ftyp', 4, 'latin1'); + buf.write('heic', 8, 'latin1'); + buf.write('heic', 16, 'latin1'); + return buf; + } + + function tinyJpeg(width: number, height: number): Buffer { + return Buffer.from([ + 0xff, 0xd8, + 0xff, 0xc0, 0x00, 0x11, 0x08, + (height >> 8) & 0xff, height & 0xff, + (width >> 8) & 0xff, width & 0xff, + 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, + 0xff, 0xd9, + ]); + } + + interface SavedFile { + readonly id: string; + readonly name: string; + readonly mimeType: string | undefined; + readonly bytes: Buffer; + } + + function memoryStore(): { readonly store: IFileService; readonly saved: SavedFile[] } { + const files = new Map(); + const saved: SavedFile[] = []; + let seq = 0; + const store: IFileService = { + _serviceBrand: undefined, + async save(source, filename, options) { + const chunks: Buffer[] = []; + for await (const chunk of source) chunks.push(Buffer.from(chunk as Uint8Array)); + const bytes = Buffer.concat(chunks); + seq += 1; + const meta: FileMeta = { + id: `f_${seq}`, + name: filename, + media_type: options?.mimeType ?? 'application/octet-stream', + size: bytes.length, + created_at: new Date(0).toISOString(), + }; + files.set(meta.id, { meta, bytes }); + saved.push({ id: meta.id, name: filename, mimeType: options?.mimeType, bytes }); + return meta; + }, + async get(fileId) { + const file = files.get(fileId); + if (file === undefined) throw new Error(`missing ${fileId}`); + return { meta: file.meta, stream: () => Readable.from([file.bytes]) }; + }, + async delete(fileId) { + files.delete(fileId); + }, + }; + return { store, saved }; + } + + const HEIC = heicBytes(); + const JPEG = tinyJpeg(6, 4); + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'kap-heic-unit-')); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + function textOf(part: unknown): string { + expect(part).toMatchObject({ type: 'text' }); + return (part as { text: string }).text; + } + + it('converts an inline base64 HEIC and sends the JPEG with a caption pointing at the saved original', async () => { + const transcode = vi.fn(async () => ({ data: JPEG, mimeType: 'image/jpeg' })); + const { store } = memoryStore(); + + const prepared = await resolvePromptMediaFiles( + [ + { + type: 'image', + name: 'IMG_0001.heic', + source: { kind: 'base64', media_type: 'image/heic', data: HEIC.toString('base64') }, + }, + ], + store, + join(dir, 'cache'), + { transcodeImage: transcode, resolveOriginalsDir: async () => join(dir, 'originals') }, + ); + + expect(transcode).toHaveBeenCalledWith(HEIC, 'image/heic'); + expect(prepared.attachments).toEqual([]); + expect(prepared.content).toHaveLength(2); + const caption = textOf(prepared.content[0]); + expect(caption).toContain('image/heic'); + expect(caption).toContain('image/jpeg'); + const saved = /saved at "([^"]+)"/.exec(caption)?.[1]; + expect(saved?.endsWith('.heic')).toBe(true); + expect(await readFile(saved!)).toEqual(HEIC); + expect(prepared.content[1]).toEqual({ + type: 'image', + name: 'IMG_0001.heic', + source: { kind: 'base64', media_type: 'image/jpeg', data: JPEG.toString('base64') }, + }); + }); + + it('converts a path-referenced HEIC and stores the JPEG as a daemon file', async () => { + const sourcePath = join(dir, 'IMG_0002.heic'); + await writeFile(sourcePath, HEIC); + const transcode = vi.fn(async () => ({ data: JPEG, mimeType: 'image/jpeg' })); + const memory = memoryStore(); + + const prepared = await resolvePromptMediaFiles( + [{ type: 'image', source: { kind: 'path', path: sourcePath } }], + memory.store, + join(dir, 'cache'), + { transcodeImage: transcode }, + ); + + expect(transcode).toHaveBeenCalledWith(HEIC, 'image/heic'); + expect(memory.saved).toEqual([ + { id: 'f_1', name: 'IMG_0002.jpeg', mimeType: 'image/jpeg', bytes: JPEG }, + ]); + expect(prepared.attachments).toEqual([]); + expect(textOf(prepared.content[0])).toContain(`saved at "${sourcePath}"`); + expect(prepared.content[1]).toEqual({ + type: 'image', + source: { kind: 'url', url: 'kimi-file://f_1' }, + name: 'IMG_0002.heic', + }); + }); + + it('converts an uploaded HEIC file and stores the JPEG as a new daemon file', async () => { + const memory = memoryStore(); + const uploaded = await memory.store.save(Readable.from([HEIC]), 'IMG_0003.heic', { + mimeType: 'image/heic', + }); + const transcode = vi.fn(async () => ({ data: JPEG, mimeType: 'image/jpeg' })); + + const prepared = await resolvePromptMediaFiles( + [{ type: 'image', source: { kind: 'file', file_id: uploaded.id } }], + memory.store, + join(dir, 'cache'), + { transcodeImage: transcode, resolveOriginalsDir: async () => join(dir, 'originals') }, + ); + + expect(transcode).toHaveBeenCalledWith(HEIC, 'image/heic'); + expect(memory.saved.at(-1)).toEqual({ + id: 'f_2', + name: 'IMG_0003.jpeg', + mimeType: 'image/jpeg', + bytes: JPEG, + }); + expect(prepared.attachments).toEqual([]); + const caption = textOf(prepared.content[0]); + expect(caption).toContain('image/heic'); + expect(/saved at "([^"]+\.heic)"/.test(caption)).toBe(true); + expect(prepared.content[1]).toEqual({ + type: 'image', + source: { kind: 'url', url: 'kimi-file://f_2' }, + name: 'IMG_0003.heic', + }); + }); + + it('keeps the attachment notice when no converter handles the HEIC', async () => { + const memory = memoryStore(); + + const prepared = await resolvePromptMediaFiles( + [ + { + type: 'image', + name: 'IMG_0004.heic', + source: { kind: 'base64', media_type: 'image/heic', data: HEIC.toString('base64') }, + }, + ], + memory.store, + join(dir, 'cache'), + { transcodeImage: async () => null, resolveAttachmentsDir: async () => join(dir, 'attachments') }, + ); + + expect(prepared.content).toHaveLength(1); + expect(textOf(prepared.content[0])).toContain('Attached file "IMG_0004.heic"'); + expect(prepared.attachments).toHaveLength(1); + expect(memory.saved).toEqual([]); + }); +});