From a5e8ec448d78f3eccca25a8d2d79dce97bc96ae0 Mon Sep 17 00:00:00 2001 From: wangxin Date: Sat, 5 Sep 2026 14:50:34 +0800 Subject: [PATCH] feat(tui): ambiguous-width mode + circled-digit display insurance (#3302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root fix — pi-tui's grapheme width now honors a configurable East Asian Ambiguous width mode: setAmbiguousWidthMode('wide'|'narrow'), surfaced as ambiguous_width in tui.toml ('auto' defaults CJK locales to wide, matching what their terminals actually render). Switching modes clears the string width cache so stale widths are never served. graphemeWidth routes through the mode-aware cellWidth. Insurance — assistant and thinking text pass through replaceCircledNumbers() before display: ①-⑳ ⑴-⒇ ⒈-⒛ ⓵-⓾ ❶-❿ ⓪⓿ all paint as '1.' '2.' '3.' forms. Display-layer only; transcript data is untouched, so /copy and logs keep the original glyphs. --- apps/kimi-code/src/cli/run-shell.ts | 13 +++++ .../components/messages/assistant-message.ts | 4 +- .../src/tui/components/messages/thinking.ts | 4 +- apps/kimi-code/src/tui/config.ts | 8 +++ apps/kimi-code/src/tui/utils/text-sanitize.ts | 42 ++++++++++++++++ apps/kimi-code/test/tui/config.test.ts | 6 +++ .../test/tui/utils/text-sanitize.test.ts | 27 ++++++++++ packages/pi-tui/src/index.ts | 3 ++ packages/pi-tui/src/utils.ts | 36 +++++++++++++- packages/pi-tui/test/ambiguous-width.test.ts | 49 +++++++++++++++++++ 10 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 apps/kimi-code/src/tui/utils/text-sanitize.ts create mode 100644 apps/kimi-code/test/tui/utils/text-sanitize.test.ts create mode 100644 packages/pi-tui/test/ambiguous-width.test.ts diff --git a/apps/kimi-code/src/cli/run-shell.ts b/apps/kimi-code/src/cli/run-shell.ts index dc32fe2af1a..b0502fa3759 100644 --- a/apps/kimi-code/src/cli/run-shell.ts +++ b/apps/kimi-code/src/cli/run-shell.ts @@ -26,6 +26,7 @@ import { CHROME_GUTTER } from '#/tui/constant/rendering'; import { KimiTUI } from '#/tui/index'; import { startupTrace } from '#/utils/startup-trace'; import { currentTheme, getColorPalette } from '#/tui/theme'; +import { setAmbiguousWidthMode } from '@moonshot-ai/pi-tui'; import { toTerminalHyperlink } from '#/utils/terminal-hyperlink'; import { restoreTerminalModes } from '#/utils/terminal-restore'; import { resolveCommandPath } from '#/utils/process/resolve-command'; @@ -57,6 +58,18 @@ export async function runShell( const palette = await getColorPalette(tuiConfig.theme); currentTheme.setPalette(palette); + // East Asian Ambiguous width mode (upstream #3302): "wide"/"narrow" honor + // tui.toml verbatim; "auto" treats CJK locales as wide, matching what their + // terminals actually render. + const ambiguous = tuiConfig.ambiguousWidth ?? 'auto'; + const ambiguousMode = + ambiguous === 'auto' + ? /^zh|ja|ko/i.test(Intl.DateTimeFormat().resolvedOptions().locale ?? '') + ? 'wide' + : 'narrow' + : ambiguous; + setAmbiguousWidthMode(ambiguousMode); + const workDir = process.cwd(); const telemetryBootstrap = createCliTelemetryBootstrap(); const telemetryClient: TelemetryClient = { diff --git a/apps/kimi-code/src/tui/components/messages/assistant-message.ts b/apps/kimi-code/src/tui/components/messages/assistant-message.ts index 64ed6bbf8d8..bafd59e3cc1 100644 --- a/apps/kimi-code/src/tui/components/messages/assistant-message.ts +++ b/apps/kimi-code/src/tui/components/messages/assistant-message.ts @@ -12,6 +12,7 @@ import { STATUS_BULLET } from '#/tui/constant/symbols'; import { currentTheme } from '#/tui/theme'; import { createMarkdownTheme } from '#/tui/theme/pi-tui-theme'; import { createMarkdownOptions } from '#/tui/utils/markdown-options'; +import { replaceCircledNumbers } from '#/tui/utils/text-sanitize'; import { markOsc133Zone } from '#/tui/utils/osc133'; import { isRenderCacheEnabled } from '#/tui/utils/render-cache'; @@ -45,7 +46,8 @@ export class AssistantMessageComponent implements Component { } updateContent(text: string, opts?: AssistantMarkdownOptions): void { - const displayText = text.trim(); + // Display-layer only: circled digits overlap on CJK terminals (#3302). + const displayText = replaceCircledNumbers(text.trim()); const transient = opts?.transient === true; if (displayText === this.lastText && transient === this.lastTransient) return; diff --git a/apps/kimi-code/src/tui/components/messages/thinking.ts b/apps/kimi-code/src/tui/components/messages/thinking.ts index e1208b55fe4..14998a7e2a6 100644 --- a/apps/kimi-code/src/tui/components/messages/thinking.ts +++ b/apps/kimi-code/src/tui/components/messages/thinking.ts @@ -16,6 +16,7 @@ import { import { STATUS_BULLET } from '#/tui/constant/symbols'; import { currentTheme } from '#/tui/theme'; import { isRenderCacheEnabled } from '#/tui/utils/render-cache'; +import { replaceCircledNumbers } from '#/tui/utils/text-sanitize'; export type ThinkingRenderMode = 'live' | 'finalized'; @@ -68,7 +69,8 @@ export class ThinkingComponent implements Component { } private styled(text: string): string { - return currentTheme.italicFg('textDim', text); + // Display-layer only: circled digits overlap on CJK terminals (#3302). + return currentTheme.italicFg('textDim', replaceCircledNumbers(text)); } finalize(): void { diff --git a/apps/kimi-code/src/tui/config.ts b/apps/kimi-code/src/tui/config.ts index 5a08af8ff77..034426f18a1 100644 --- a/apps/kimi-code/src/tui/config.ts +++ b/apps/kimi-code/src/tui/config.ts @@ -56,6 +56,9 @@ export const TuiConfigFileSchema = z.object({ render_latex: z.boolean().optional(), disable_paste_burst: z.boolean().optional(), cache_expiry_hint: z.boolean().optional(), + /** East Asian Ambiguous chars (① ★ →) cell width: "narrow"=1, "wide"=2, + * "auto"=detect from locale (CJK locales default wide; upstream #3302). */ + ambiguous_width: z.enum(['narrow', 'wide', 'auto']).optional(), editor: z .object({ command: z.string().optional(), @@ -84,6 +87,9 @@ export const TuiConfigSchema = z.object({ /** Present in every normalized config; optional only so hand-built test * fixtures from before this field existed still typecheck. */ cacheExpiryHint: z.boolean().optional(), + /** Resolved cell width for East Asian Ambiguous chars; "auto" defers to + * locale detection at application time. */ + ambiguousWidth: z.enum(['narrow', 'wide', 'auto']).optional(), editorCommand: z.string().nullable(), notifications: NotificationsConfigSchema, upgrade: UpgradePreferencesSchema, @@ -111,6 +117,7 @@ export const DEFAULT_TUI_CONFIG: TuiConfig = TuiConfigSchema.parse({ renderLatex: true, disablePasteBurst: false, cacheExpiryHint: true, + ambiguousWidth: 'auto', editorCommand: null, notifications: DEFAULT_NOTIFICATIONS_CONFIG, upgrade: DEFAULT_UPGRADE_PREFERENCES, @@ -198,6 +205,7 @@ export function normalizeTuiConfig( renderLatex: config.render_latex ?? DEFAULT_TUI_CONFIG.renderLatex, disablePasteBurst: config.disable_paste_burst ?? DEFAULT_TUI_CONFIG.disablePasteBurst, cacheExpiryHint: config.cache_expiry_hint ?? DEFAULT_TUI_CONFIG.cacheExpiryHint, + ambiguousWidth: config.ambiguous_width ?? DEFAULT_TUI_CONFIG.ambiguousWidth, editorCommand: command === undefined || command.length === 0 ? null : command, notifications: { enabled: config.notifications?.enabled ?? DEFAULT_NOTIFICATIONS_CONFIG.enabled, diff --git a/apps/kimi-code/src/tui/utils/text-sanitize.ts b/apps/kimi-code/src/tui/utils/text-sanitize.ts new file mode 100644 index 00000000000..5f478dd705d --- /dev/null +++ b/apps/kimi-code/src/tui/utils/text-sanitize.ts @@ -0,0 +1,42 @@ +/** + * Display-layer sanitizer for assistant-facing text. + * + * Circled/parenthesized digit glyphs (①-⑳ ❶-❿ ⓵-⓾ ⓪⓿) are East Asian + * Ambiguous: CJK terminals render them double-width, so a line that mixes them + * with single-width digits overlaps and garbles (upstream kimi-code #3302). + * The width-mode fix (ambiguous_width) only helps when the terminal agrees; + * when it doesn't, these glyphs are still visually fragile across fonts. + * Displaying "1." instead never misaligns — the underlying transcript data is + * untouched, this only rewrites what is painted. + */ + +const CIRCLED_DIGIT_MAP: ReadonlyMap = (() => { + const map = new Map(); + // ①-⑳ U+2460..U+2473 → 1..20 (circled) + for (let i = 0; i < 20; i++) map.set(0x2460 + i, `${i + 1}.`); + // ⑴-⒇ U+2474..U+2487 → 1..20 (parenthesized) + for (let i = 0; i < 20; i++) map.set(0x2474 + i, `${i + 1}.`); + // ⒈-⒛ U+2488..U+249B → 1..20 (digit + period glyph) + for (let i = 0; i < 20; i++) map.set(0x2488 + i, `${i + 1}.`); + // ⓵-⓾ U+24F5..U+24FE → 1..10 (double-circled) + for (let i = 0; i < 10; i++) map.set(0x24f5 + i, `${i + 1}.`); + // ❶-❿ U+2776..U+277F → 1..10 (dingbat negative circled) + for (let i = 0; i < 10; i++) map.set(0x2776 + i, `${i + 1}.`); + // ⓪ U+24EA, ⓿ U+24FF → 0. + map.set(0x24ea, '0.'); + map.set(0x24ff, '0.'); + return map; +})(); + +/** Replace circled/parenthesized digit glyphs with "N." display forms. */ +export function replaceCircledNumbers(text: string): string { + // Fast path: bail before iterating code points. + if (!/[①-⑳⑴-⒇⒈-⒛⓪⓵-⓾⓿❶-❿]/.test(text)) return text; + let out = ''; + for (const ch of text) { + const cp = ch.codePointAt(0)!; + const replacement = CIRCLED_DIGIT_MAP.get(cp); + out += replacement ?? ch; + } + return out; +} diff --git a/apps/kimi-code/test/tui/config.test.ts b/apps/kimi-code/test/tui/config.test.ts index 48df4730307..441623d9318 100644 --- a/apps/kimi-code/test/tui/config.test.ts +++ b/apps/kimi-code/test/tui/config.test.ts @@ -63,6 +63,7 @@ auto_install = false renderLatex: true, disablePasteBurst: false, cacheExpiryHint: true, + ambiguousWidth: 'auto', editorCommand: 'code --wait', notifications: { enabled: false, condition: 'always' }, upgrade: { autoInstall: false }, @@ -109,6 +110,7 @@ command = " " renderLatex: true, disablePasteBurst: false, cacheExpiryHint: true, + ambiguousWidth: 'auto', editorCommand: null, notifications: { enabled: true, condition: 'unfocused' }, upgrade: { autoInstall: true }, @@ -143,6 +145,8 @@ command = " " theme: 'light', disablePasteBurst: false, cacheExpiryHint: true, + ambiguousWidth: 'auto', + ambiguousWidth: 'auto', editorCommand: 'vim', notifications: { enabled: false, condition: 'always' }, upgrade: { autoInstall: false }, @@ -156,6 +160,7 @@ command = " " renderLatex: true, disablePasteBurst: false, cacheExpiryHint: true, + ambiguousWidth: 'auto', editorCommand: 'vim', notifications: { enabled: false, condition: 'always' }, upgrade: { autoInstall: false }, @@ -170,6 +175,7 @@ command = " " theme, disablePasteBurst: DEFAULT_TUI_CONFIG.disablePasteBurst, cacheExpiryHint: DEFAULT_TUI_CONFIG.cacheExpiryHint, + ambiguousWidth: DEFAULT_TUI_CONFIG.ambiguousWidth, editorCommand: null, notifications: DEFAULT_TUI_CONFIG.notifications, upgrade: DEFAULT_TUI_CONFIG.upgrade, diff --git a/apps/kimi-code/test/tui/utils/text-sanitize.test.ts b/apps/kimi-code/test/tui/utils/text-sanitize.test.ts new file mode 100644 index 00000000000..9cef01ac32a --- /dev/null +++ b/apps/kimi-code/test/tui/utils/text-sanitize.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; + +import { replaceCircledNumbers } from '#/tui/utils/text-sanitize'; + +describe('replaceCircledNumbers (#3302 display-layer insurance)', () => { + it('replaces ①-⑳ with 1.-20.', () => { + expect(replaceCircledNumbers('①②③')).toBe('1.2.3.'); + expect(replaceCircledNumbers('第⑳项')).toBe('第20.项'); + }); + + it('replaces ❶-❿ and ⓵-⓾ and zero forms', () => { + expect(replaceCircledNumbers('❶❿')).toBe('1.10.'); + // ⓵⓾ are double-circled 1 and 10 (U+24F5/U+24FE) + expect(replaceCircledNumbers('⓵⓾')).toBe('1.10.'); + expect(replaceCircledNumbers('⑴⒇')).toBe('1.20.'); + expect(replaceCircledNumbers('⓪⓿')).toBe('0.0.'); + }); + + it('leaves ordinary text untouched', () => { + expect(replaceCircledNumbers('plain ASCII 123')).toBe('plain ASCII 123'); + expect(replaceCircledNumbers('中文没有圈号')).toBe('中文没有圈号'); + }); + + it('handles the user regression probe', () => { + expect(replaceCircledNumbers('①测试 ★测试 →测试 α测试')).toBe('1.测试 ★测试 →测试 α测试'); + }); +}); diff --git a/packages/pi-tui/src/index.ts b/packages/pi-tui/src/index.ts index 0d5a4a1093b..59535b4c2e3 100644 --- a/packages/pi-tui/src/index.ts +++ b/packages/pi-tui/src/index.ts @@ -138,10 +138,13 @@ export { TuiAltScreen, type TuiAltScreenOptions } from "./tui-alt-screen.ts"; export { TuiMainScreen, type TuiMainScreenRenderState } from "./tui-main-screen.ts"; // Utilities export { + getAmbiguousWidthMode, getOsc8LinkAtColumn, + setAmbiguousWidthMode, sliceByColumn, stripTerminalSequences, truncateToWidth, visibleWidth, wrapTextWithAnsi, + type AmbiguousWidthMode, } from "./utils.ts"; diff --git a/packages/pi-tui/src/utils.ts b/packages/pi-tui/src/utils.ts index 2d82c51acdc..5b056939fd5 100644 --- a/packages/pi-tui/src/utils.ts +++ b/packages/pi-tui/src/utils.ts @@ -1,5 +1,37 @@ import { eastAsianWidth } from "get-east-asian-width"; +// --------------------------------------------------------------------------- +// East Asian Ambiguous width mode +// --------------------------------------------------------------------------- + +/** + * Whether East Asian Ambiguous characters (① ★ → α …) count as 1 or 2 + * terminal cells. Terminals in CJK locales render them double-width; treating + * them as narrow misaligns every padded line that contains one (upstream + * kimi-code #3302). Default "narrow" matches upstream; the host sets "wide" + * from `ambiguous_width` in tui.toml (auto → CJK locale detection). + */ +export type AmbiguousWidthMode = "narrow" | "wide"; + +let ambiguousWidthMode: AmbiguousWidthMode = "narrow"; + +export function setAmbiguousWidthMode(mode: AmbiguousWidthMode): void { + if (mode === ambiguousWidthMode) return; + ambiguousWidthMode = mode; + // widthCache (below) is keyed by string only — a mode switch must not serve + // widths computed under the old mode. + widthCache.clear(); +} + +export function getAmbiguousWidthMode(): AmbiguousWidthMode { + return ambiguousWidthMode; +} + +/** eastAsianWidth honoring the configured ambiguous-width mode. */ +function cellWidth(cp: number): number { + return eastAsianWidth(cp, { ambiguousAsWide: ambiguousWidthMode === "wide" }); +} + // segmenters (shared instance) const graphemeSegmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" }); const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); @@ -205,7 +237,7 @@ function graphemeWidth(segment: string): number { return 2; } - let width = eastAsianWidth(cp); + let width = cellWidth(cp); // Intl.Segmenter can group multiple terminal-spacing code points into one // grapheme. Count trailing visible code points that terminals may allocate @@ -223,7 +255,7 @@ function graphemeWidth(segment: string): number { const c = char.codePointAt(0)!; if (followsMark || (c >= 0xff00 && c <= 0xffef)) { // halfwidth + fullwidth forms - width += eastAsianWidth(c); + width += cellWidth(c); } else if (c === 0x0e33 || c === 0x0eb3) { width += 1; } diff --git a/packages/pi-tui/test/ambiguous-width.test.ts b/packages/pi-tui/test/ambiguous-width.test.ts new file mode 100644 index 00000000000..fb45260df9a --- /dev/null +++ b/packages/pi-tui/test/ambiguous-width.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { + getAmbiguousWidthMode, + setAmbiguousWidthMode, + visibleWidth, +} from "../src/utils.ts"; + +describe("ambiguous width mode (upstream #3302)", () => { + it("defaults to narrow", () => { + setAmbiguousWidthMode("narrow"); + assert.strictEqual(getAmbiguousWidthMode(), "narrow"); + assert.strictEqual(visibleWidth("①"), 1); + }); + + it("treats East Asian Ambiguous chars as 2 cells in wide mode", () => { + setAmbiguousWidthMode("wide"); + try { + // ① circled digit, ★ star, → arrow, α greek — all Ambiguous class + assert.strictEqual(visibleWidth("①"), 2); + assert.strictEqual(visibleWidth("★"), 2); + assert.strictEqual(visibleWidth("→"), 2); + assert.strictEqual(visibleWidth("α"), 2); + // CJK ideographs were always wide; unaffected by the mode + assert.strictEqual(visibleWidth("汉"), 2); + // plain ASCII stays 1 + assert.strictEqual(visibleWidth("a"), 1); + } finally { + setAmbiguousWidthMode("narrow"); + } + }); + + it("padded columns align when mixing circled digits and CJK in wide mode", () => { + setAmbiguousWidthMode("wide"); + try { + // The regression shape from #3302: a line whose ambiguous glyphs were + // undercounted by 1 cell each wrapped/overlapped its neighbor. + const a = "①测试"; + const b = "1.测试"; + assert.strictEqual(visibleWidth(a), visibleWidth("1.") + visibleWidth("测试")); + assert.strictEqual(visibleWidth(b), visibleWidth("1.") + visibleWidth("测试")); + // The user's regression probe: ①测试 ★测试 →测试 α测试 — every token + // must sum to its true cell count so no padding overlap can occur. + assert.strictEqual(visibleWidth("★测试 →测试 α测试"), 2 + 4 + 1 + 2 + 4 + 1 + 2 + 4); + } finally { + setAmbiguousWidthMode("narrow"); + } + }); +});