Skip to content

feat(media): convert HEIC images to JPEG with sips on macOS #3647

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RealKai42 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(media): convert HEIC images to JPEG with sips on macOS #3647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/heic-macos-sips.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": minor
---

Read and attach HEIC/HEIF images on macOS; they are converted to JPEG automatically, including files pasted into the prompt.
81 changes: 63 additions & 18 deletions apps/kimi-code/src/utils/clipboard/clipboard-image.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
* kimi-core's LLM pipeline only accepts PNG/JPEG/GIF/WebP, and the
* clipboard sources we query already emit those formats on supported
* platforms — so we deliberately do not include a BMP→PNG converter.
* The one exception is a copied HEIC/HEIF *file* on macOS (iPhone photos
* and screenshots): it is converted to JPEG through the system `sips`
* before being pasted. Elsewhere such a file is still declined.
*
* Lookup order:
* macOS file clipboard -> osascript/AppKit file URLs
Expand All @@ -22,7 +25,7 @@ import { tmpdir } from 'node:os';
import { basename, isAbsolute, join } from 'node:path';
import { fileURLToPath } from 'node:url';

import { parseImageMeta } from '#/utils/image/image-mime';
import { isHeicImage, parseImageMeta } from '#/utils/image/image-mime';

import {
DEFAULT_LIST_TIMEOUT_MS,
Expand Down Expand Up @@ -82,6 +85,13 @@ const VIDEO_MIME_BY_SUFFIX: Readonly<Record<string, string>> = Object.freeze({

const DEFAULT_READ_TIMEOUT_MS = 3000;
const DEFAULT_POWERSHELL_TIMEOUT_MS = 5000;
const SIPS_TIMEOUT_MS = 15_000;

/** Platform facts a clipboard file path needs to be turned into pasteable media. */
interface PathReadContext {
readonly run: RunCommand;
readonly platform: NodeJS.Platform;
}

const MACOS_FILE_PATH_SCRIPT = String.raw`
ObjC.import('AppKit');
Expand Down Expand Up @@ -170,7 +180,7 @@ function splitClipboardPathLines(text: string): string[] {
return lines;
}

function readImagePath(path: string): ClipboardImage | null {
function readImagePath(path: string, ctx: PathReadContext): ClipboardImage | null {
let stat: ReturnType<typeof statSync>;
try {
stat = statSync(path);
Expand All @@ -187,11 +197,44 @@ function readImagePath(path: string): ClipboardImage | null {
}
if (bytes.length === 0) return null;

const meta = parseImageMeta(bytes);
let meta = parseImageMeta(bytes);
if (meta === null && ctx.platform === 'darwin' && isHeicImage(bytes)) {
const converted = convertHeicViaSips(path, ctx.run);
if (converted === null) return null;
bytes = converted;
meta = parseImageMeta(bytes);
}
if (meta === null) return null;
return { kind: 'image', bytes: new Uint8Array(bytes), mimeType: meta.mime };
}

/**
* macOS ships `sips`, which decodes HEIC with the system codecs. The JPEG
* lands in a temp file (sips has no stdout mode) that is removed on every
* exit path; any failure declines the paste instead of surfacing an error.
*/
function convertHeicViaSips(path: string, run: RunCommand): Buffer | null {
const target = join(tmpdir(), `kimi-heic-${randomUUID()}.jpg`);
try {
const result = run(
'sips',
['-s', 'format', 'jpeg', '-s', 'formatOptions', '90', path, '--out', target],
{ timeoutMs: SIPS_TIMEOUT_MS },
Comment on lines +219 to +222

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 8, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Run HEIC conversion without blocking the TUI

When sips is slow or wedged, this calls the default synchronous runCommand implementation, which uses spawnSync, from the explicit paste handler before the attachment placeholder is inserted. The entire Node event loop therefore stops processing input, rendering, and cancellation for as long as the 15-second timeout; run the conversion asynchronously or move it into the existing background image-ingestion work.

Useful? React with 👍 / 👎.

);
if (!result.ok) return null;
const bytes = readFileSync(target);
return bytes.length === 0 ? null : bytes;
} catch {
return null;
} finally {
try {
unlinkSync(target);
} catch {
// ignore cleanup errors
}
}
}

function readVideoPath(path: string): ClipboardVideo | null {
const mimeType = videoMimeFromPath(path);
if (mimeType === null) return null;
Expand All @@ -215,23 +258,23 @@ function readVideoPath(path: string): ClipboardVideo | null {
};
}

function readMediaPath(path: string): ClipboardMedia | null {
function readMediaPath(path: string, ctx: PathReadContext): ClipboardMedia | null {
// Video files are never opened as images.
const video = readVideoPath(path);
if (video !== null) return video;
return readImagePath(path);
return readImagePath(path, ctx);
}

function readMediaFromPaths(paths: readonly string[]): ClipboardMedia | null {
function readMediaFromPaths(paths: readonly string[], ctx: PathReadContext): ClipboardMedia | null {
for (const path of paths) {
const media = readMediaPath(path);
const media = readMediaPath(path, ctx);
if (media !== null) return media;
}
return null;
}

function readMediaFromText(text: string): ClipboardMedia | null {
return readMediaFromPaths(parseClipboardPaths(text));
function readMediaFromText(text: string, ctx: PathReadContext): ClipboardMedia | null {
return readMediaFromPaths(parseClipboardPaths(text), ctx);
}

function runCommand(command: string, args: string[], options?: RunCommandOptions): { stdout: Buffer; ok: boolean } {
Expand All @@ -241,7 +284,7 @@ function runCommand(command: string, args: string[], options?: RunCommandOptions
});
}

function readClipboardFileMediaViaWlPaste(): ClipboardMedia | null {
function readClipboardFileMediaViaWlPaste(ctx: PathReadContext): ClipboardMedia | null {
const list = runCommand('wl-paste', ['--list-types'], {
timeoutMs: DEFAULT_LIST_TIMEOUT_MS,
});
Expand All @@ -252,7 +295,7 @@ function readClipboardFileMediaViaWlPaste(): ClipboardMedia | null {
if (uriType === undefined) return null;

const uris = runCommand('wl-paste', ['--type', uriType, '--no-newline']);
return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8')) : null;
return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8'), ctx) : null;
}

function readClipboardImageViaWlPaste(): ClipboardImage | null {
Expand All @@ -269,7 +312,7 @@ function readClipboardImageViaWlPaste(): ClipboardImage | null {
return { kind: 'image', bytes: data.stdout, mimeType: baseMimeType(selected) };
}

function readClipboardFileMediaViaXclip(): ClipboardMedia | null {
function readClipboardFileMediaViaXclip(ctx: PathReadContext): ClipboardMedia | null {
const targets = runCommand('xclip', ['-selection', 'clipboard', '-t', 'TARGETS', '-o'], {
timeoutMs: DEFAULT_LIST_TIMEOUT_MS,
});
Expand All @@ -280,7 +323,7 @@ function readClipboardFileMediaViaXclip(): ClipboardMedia | null {
if (uriType === undefined) return null;

const uris = runCommand('xclip', ['-selection', 'clipboard', '-t', uriType, '-o']);
return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8')) : null;
return uris.ok ? readMediaFromText(uris.stdout.toString('utf-8'), ctx) : null;
}

function readClipboardImageViaXclip(): ClipboardImage | null {
Expand Down Expand Up @@ -359,6 +402,7 @@ function readClipboardFilePathsViaMacOs(run: RunCommand): string[] {

async function readClipboardFileMediaViaNativeText(
clip: ClipboardModule | null,
ctx: PathReadContext,
): Promise<{ media: ClipboardMedia | null; lookedFileLike: boolean }> {
if (clip === null) return { media: null, lookedFileLike: false };

Expand All @@ -369,7 +413,7 @@ async function readClipboardFileMediaViaNativeText(
}

try {
return { media: readMediaFromText(await clip.getText()), lookedFileLike };
return { media: readMediaFromText(await clip.getText(), ctx), lookedFileLike };
} catch (error) {
if (error instanceof ClipboardMediaError) throw error;
return { media: null, lookedFileLike };
Expand Down Expand Up @@ -408,6 +452,7 @@ export async function readClipboardMedia(options?: {
const platform = options?.platform ?? process.platform;
const clip = options?.clipboard ?? clipboard;
const run = options?.runCommand ?? runCommand;
const ctx: PathReadContext = { run, platform };

// Termux on Android has no desktop clipboard; skip early rather than
// churn through every fallback.
Expand All @@ -419,26 +464,26 @@ export async function readClipboardMedia(options?: {
const wsl = isWSL(env);

if (wayland || wsl) {
const fileMedia = readClipboardFileMediaViaWlPaste() ?? readClipboardFileMediaViaXclip();
const fileMedia = readClipboardFileMediaViaWlPaste(ctx) ?? readClipboardFileMediaViaXclip(ctx);
if (fileMedia !== null) return fileMedia;
image = readClipboardImageViaWlPaste() ?? readClipboardImageViaXclip();
}
if (image === null && wsl) {
image = readClipboardImageViaPowerShell();
}
if (image === null && !wayland) {
const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip);
const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip, ctx);
if (nativeFileMedia.media !== null) return nativeFileMedia.media;
if (nativeFileMedia.lookedFileLike) return null;
image = await readClipboardImageViaNative(clip);
}
} else {
if (platform === 'darwin') {
const fileMedia = readMediaFromPaths(readClipboardFilePathsViaMacOs(run));
const fileMedia = readMediaFromPaths(readClipboardFilePathsViaMacOs(run), ctx);
if (fileMedia !== null) return fileMedia;
}

const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip);
const nativeFileMedia = await readClipboardFileMediaViaNativeText(clip, ctx);
if (nativeFileMedia.media !== null) return nativeFileMedia.media;

// Finder exposes file icons/thumbnails as image data. If the clipboard
Expand Down
19 changes: 19 additions & 0 deletions apps/kimi-code/src/utils/image/image-mime.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,25 @@ export function parseImageMeta(bytes: Uint8Array): ImageMeta | null {
return null;
}

// ── HEIC / HEIF ─────────────────────────────────────────────────────

const HEIC_FTYP_BRANDS = new Set(['heic', 'heix', 'hevc', 'hevx', 'heif', 'mif1', 'msf1']);

/**
* Sniff an ISO-BMFF `ftyp` box whose major brand is one of the HEIF
* family. HEIC is not a format the model pipeline accepts, so this is
* only used to decide whether a pasted file should be converted first
* (macOS `sips`) rather than declined outright.
*/
export function isHeicImage(bytes: Uint8Array): boolean {
if (bytes.length < 12) return false;
if (bytes[4] !== 0x66 || bytes[5] !== 0x74 || bytes[6] !== 0x79 || bytes[7] !== 0x70) return false;
const brand = String.fromCodePoint(bytes[8]!, bytes[9]!, bytes[10]!, bytes[11]!)
.trim()
.toLowerCase();
return HEIC_FTYP_BRANDS.has(brand);
}

// ── PNG ─────────────────────────────────────────────────────────────

function isPng(b: Uint8Array): boolean {
Expand Down
117 changes: 116 additions & 1 deletion apps/kimi-code/test/utils/clipboard/clipboard-image.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs';
import { existsSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
Expand Down Expand Up @@ -36,6 +36,42 @@ function noMacOsPaths(): { stdout: Buffer; ok: boolean } {
return { stdout: Buffer.alloc(0), ok: false };
}

function heic(): Uint8Array {
const bytes = new Uint8Array(24);
bytes.set([0x00, 0x00, 0x00, 0x18], 0);
bytes.set([0x66, 0x74, 0x79, 0x70], 4);
bytes.set([0x68, 0x65, 0x69, 0x63], 8);
bytes.set([0x68, 0x65, 0x69, 0x63], 16);
return bytes;
}

function jpeg(width: number, height: number): Uint8Array {
return new Uint8Array([
0xff, 0xd8,
0xff, 0xc0, 0x00, 0x11, 0x08,
(height >> 8) & 0xff, height & 0xff,
(width >> 8) & 0xff, width & 0xff,
0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01,
0xff, 0xd9,
]);
}

type CommandResult = { stdout: Buffer; ok: boolean };

function macCommands(options: {
readonly paths: string;
readonly sips?: (args: string[]) => CommandResult;
}): { run: (command: string, args: string[]) => CommandResult; calls: string[][] } {
const calls: string[][] = [];
const run = (command: string, args: string[]): CommandResult => {
calls.push([command, ...args]);
if (command === 'osascript') return { stdout: Buffer.from(options.paths), ok: true };
if (command === 'sips' && options.sips !== undefined) return options.sips(args);
return { stdout: Buffer.alloc(0), ok: false };
};
return { run, calls };
}

describe('readClipboardMedia', () => {
it('reads a copied image file from its real path instead of the Finder preview icon', async () => {
const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-'));
Expand Down Expand Up @@ -135,6 +171,85 @@ describe('readClipboardMedia', () => {
expect(getImageBinary).not.toHaveBeenCalled();
});

it('converts a copied HEIC file through sips on macOS and pastes the JPEG', async () => {
const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-'));
try {
const heicPath = join(dir, 'IMG_0001.HEIC');
writeFileSync(heicPath, heic());
const converted = jpeg(6, 4);
const commands = macCommands({
paths: `${heicPath}\n`,
sips: (args) => {
writeFileSync(args[args.indexOf('--out') + 1]!, converted);
return { stdout: Buffer.alloc(0), ok: true };
},
});
const clip = fakeClipboard({ availableFormats: vi.fn(() => ['public.file-url']) });

const media = await readClipboardMedia({
platform: 'darwin',
clipboard: clip,
runCommand: commands.run,
});

expect(media).toEqual({ kind: 'image', bytes: converted, mimeType: 'image/jpeg' });
const sips = commands.calls.find((call) => call[0] === 'sips');
expect(sips).toBeDefined();
expect(sips!.slice(1, 7)).toEqual(['-s', 'format', 'jpeg', '-s', 'formatOptions', '90']);
expect(sips![7]).toBe(heicPath);
expect(sips![8]).toBe('--out');
expect(sips![9]!.endsWith('.jpg')).toBe(true);
expect(existsSync(sips![9]!)).toBe(false);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it('declines the paste when sips cannot convert the copied HEIC', async () => {
const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-'));
try {
const heicPath = join(dir, 'broken.heic');
writeFileSync(heicPath, heic());
const commands = macCommands({
paths: `${heicPath}\n`,
sips: () => ({ stdout: Buffer.alloc(0), ok: false }),
});
const clip = fakeClipboard({ availableFormats: vi.fn(() => ['public.file-url']) });

const media = await readClipboardMedia({
platform: 'darwin',
clipboard: clip,
runCommand: commands.run,
});

expect(media).toBeNull();
expect(commands.calls.some((call) => call[0] === 'sips')).toBe(true);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it('does not try to convert a copied HEIC file off macOS', async () => {
const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-'));
try {
const heicPath = join(dir, 'IMG_0002.heic');
writeFileSync(heicPath, heic());
const runCommand = vi.fn(() => ({ stdout: Buffer.alloc(0), ok: false }));
const clip = fakeClipboard({
availableFormats: vi.fn(() => ['text/uri-list']),
hasText: vi.fn(() => true),
getText: vi.fn(async () => pathToFileURL(heicPath).toString()),
});

const media = await readClipboardMedia({ platform: 'win32', clipboard: clip, runCommand });

expect(media).toBeNull();
expect(runCommand).not.toHaveBeenCalled();
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it('rejects pasted videos larger than 100 MB', async () => {
const dir = mkdtempSync(join(tmpdir(), 'kimi-code-clip-'));
try {
Expand Down
Loading
Loading