diff --git a/index.js b/index.js index 42ec3a1..29fb275 100644 --- a/index.js +++ b/index.js @@ -4,6 +4,7 @@ import {eastAsianWidth} from 'get-east-asian-width'; /** Logic: - Segment graphemes to match how terminals render clusters. +- Runs of characters that never join a neighbouring cluster (ASCII, most Latin, Greek, Cyrillic, CJK, Hangul syllables) are counted without segmenting; only the text around other characters is segmented. - Width rules: 1. Skip non-printing clusters (Default_Ignorable, Control, pure nonspacing/enclosing Mark, lone Surrogates). Tabs are ignored by design. 2. RGI emoji clusters (\p{RGI_Emoji}) are double-width. @@ -48,6 +49,49 @@ function isDoubleWidthNonRgiEmojiSequence(segment) { return false; } +/* +Characters that never share a grapheme cluster with a neighbour from the same set (UAX #29 rules GB3 to GB13). +None of them is Extend, ZWJ, SpacingMark, Prepend, Regional_Indicator, Hangul jamo, Control, CR, LF or an Indic conjunct consonant or linker, +so between two of them GB999 (break everywhere) always applies. Extended_Pictographic characters qualify because joining them needs a ZWJ, +and precomposed Hangul syllables qualify because joining them needs a jamo. The test suite enumerates every code point of these ranges. +*/ +export const _standaloneRanges = [ + [0x20, 0x7E], // Printable ASCII + [0xA0, 0xAC], // Latin-1 Supplement up to the soft hyphen (U+00AD is Control) + [0xAE, 0x2_FF], // Rest of Latin-1 Supplement, Latin Extended-A and B, IPA Extensions, Spacing Modifier Letters + [0x3_70, 0x3_FF], // Greek and Coptic + [0x4_00, 0x4_82], // Cyrillic up to the combining marks (U+0483 to U+0489) + [0x4_8A, 0x5_2F], // Rest of Cyrillic, Cyrillic Supplement + [0x1E_00, 0x1F_FF], // Latin Extended Additional, Greek Extended + [0x20_00, 0x20_0A], // Spaces (U+200B to U+200F are Format) + [0x20_10, 0x20_27], // Dashes, quotes, bullets, ellipsis (U+2028 to U+202E are separators and Format) + [0x20_30, 0x20_5E], // Rest of General Punctuation (U+2060 to U+206F are Format) + [0x20_70, 0x20_CF], // Superscripts and Subscripts, Currency Symbols (U+20D0 to U+20FF are combining marks) + [0x21_00, 0x2B_FF], // Letterlike Symbols to Miscellaneous Symbols and Arrows: arrows, math, box drawing, shapes, symbols, dingbats + [0x30_00, 0x30_29], // CJK Symbols and Punctuation (U+302A to U+302F are combining marks) + [0x30_30, 0x30_98], // Rest of CJK Symbols and Punctuation, Hiragana (U+3099 and U+309A are combining marks) + [0x30_9B, 0x30_FF], // Rest of Hiragana, Katakana + [0x31_00, 0x31_63], // Bopomofo, Hangul Compatibility Jamo (U+3164 is default ignorable) + [0x31_65, 0x33_FF], // Rest of Hangul Compatibility Jamo, Kanbun, Enclosed CJK Letters and Months, CJK Compatibility + [0x34_00, 0x4D_BF], // CJK Unified Ideographs Extension A + [0x4E_00, 0x9F_FF], // CJK Unified Ideographs + [0xAC_00, 0xD7_A3], // Hangul Syllables + [0xF9_00, 0xFA_FF], // CJK Compatibility Ideographs + [0xFF_01, 0xFF_9D], // Fullwidth ASCII variants, Halfwidth Katakana (U+FF9E and U+FF9F are Extend) + [0xFF_E0, 0xFF_EE], // Fullwidth and halfwidth signs + [0x2_00_00, 0x2_FA_1F], // CJK Unified Ideographs Extensions B to F and I, CJK Compatibility Ideographs Supplement + [0x3_00_00, 0x3_23_AF], // CJK Unified Ideographs Extensions G and H +]; + +const standaloneRunRegex = new RegExp(`[${_standaloneRanges.map(([start, end]) => `\\u{${start.toString(16)}}-\\u{${end.toString(16)}}`).join('')}]+`, 'gu'); + +// Segmenting a slice costs about as much as classifying a dozen single code point clusters, so a run in the middle of the string is only skipped when it is long enough to pay for the extra slice +const minimumInnerRunLength = 16; + +function isPrintableAscii(codePoint) { + return codePoint >= 0x20 && codePoint <= 0x7E; +} + function baseVisible(segment) { return segment.replace(leadingNonPrintingRegex, ''); } @@ -147,6 +191,130 @@ function trailingWidth(visibleSegment, eastAsianWidthOptions) { return extra; } +function clusterWidth(segment, eastAsianWidthOptions) { + // Zero-width / non-printing clusters + if (isZeroWidthCluster(segment)) { + return 0; + } + + // Emoji width logic + if (rgiEmojiRegex.test(segment) || isDoubleWidthNonRgiEmojiSequence(segment)) { + return 2; + } + + const visibleSegment = baseVisible(segment); + const hangulWidth = hangulClusterWidth(visibleSegment, eastAsianWidthOptions); + if (hangulWidth !== undefined) { + return hangulWidth; + } + + // Everything else: EAW of the cluster’s first visible scalar, plus trailing spacing marks and Halfwidth/Fullwidth Forms (e.g., ﾞ, ﾟ, ｰ) + const codePoint = visibleSegment.codePointAt(0); + return eastAsianWidth(codePoint, eastAsianWidthOptions) + trailingWidth(visibleSegment, eastAsianWidthOptions); +} + +// The width of a single code point cluster depends only on the code point and the ambiguous width option, so cache it per option +const narrowCodePointWidths = new Map(); +const wideCodePointWidths = new Map(); +const maximumCachedCodePoints = 2048; + +function codePointWidth(codePoint, eastAsianWidthOptions) { + const cache = eastAsianWidthOptions.ambiguousAsWide ? wideCodePointWidths : narrowCodePointWidths; + let width = cache.get(codePoint); + + if (width === undefined) { + width = clusterWidth(String.fromCodePoint(codePoint), eastAsianWidthOptions); + + if (cache.size >= maximumCachedCodePoints) { + cache.clear(); + } + + cache.set(codePoint, width); + } + + return width; +} + +function segmentedWidth(string, eastAsianWidthOptions) { + let width = 0; + + for (const {segment} of segmenter.segment(string)) { + const codePoint = segment.codePointAt(0); + + // A lone printable ASCII character is always width 1, so skip the cluster checks + if (segment.length === 1 && isPrintableAscii(codePoint)) { + width += 1; + continue; + } + + // Single code point clusters are by far the most common and repeat a lot, so use the cache + if (segment.length === (codePoint > 0xFF_FF ? 2 : 1)) { + width += codePointWidth(codePoint, eastAsianWidthOptions); + continue; + } + + width += clusterWidth(segment, eastAsianWidthOptions); + } + + return width; +} + +function isLowSurrogate(codeUnit) { + return codeUnit >= 0xDC_00 && codeUnit <= 0xDF_FF; +} + +// Width of a run of standalone characters: every one is its own cluster +function standaloneRunWidth(string, start, end, eastAsianWidthOptions) { + let width = 0; + + for (let index = start; index < end;) { + const codePoint = string.codePointAt(index); + index += codePoint > 0xFF_FF ? 2 : 1; + width += codePoint <= 0x7E ? 1 : codePointWidth(codePoint, eastAsianWidthOptions); + } + + return width; +} + +function textWidth(string, eastAsianWidthOptions) { + const {length} = string; + let width = 0; + + // Start of the text that still has to be segmented + let pendingStart = 0; + + // Runs of standalone characters are counted one code point at a time; only the text around other + // characters is segmented. The first and last character of a run may still join a cluster with a + // neighbouring character (combining marks, ZWJ, keycaps, prepends), so they are left to the + // segmenter unless the run touches the start or end of the string. + standaloneRunRegex.lastIndex = 0; + for (let match = standaloneRunRegex.exec(string); match !== null; match = standaloneRunRegex.exec(string)) { + const runStart = match.index; + const runEnd = standaloneRunRegex.lastIndex; + // Step over whole code points: the first or last character of the run may be a surrogate pair + const skipStart = runStart === 0 ? 0 : runStart + (string.codePointAt(runStart) > 0xFF_FF ? 2 : 1); + const skipEnd = runEnd === length ? length : runEnd - (isLowSurrogate(string.codePointAt(runEnd - 1)) ? 2 : 1); + const isInnerRun = runStart > 0 && runEnd < length; + + if (skipEnd <= skipStart || (isInnerRun && skipEnd - skipStart < minimumInnerRunLength)) { + continue; + } + + if (skipStart > pendingStart) { + width += segmentedWidth(string.slice(pendingStart, skipStart), eastAsianWidthOptions); + } + + width += standaloneRunWidth(string, skipStart, skipEnd, eastAsianWidthOptions); + pendingStart = skipEnd; + } + + if (pendingStart < length) { + width += segmentedWidth(pendingStart === 0 ? string : string.slice(pendingStart), eastAsianWidthOptions); + } + + return width; +} + export default function stringWidth(input, options = {}) { if (typeof input !== 'string' || input.length === 0) { return 0; @@ -173,35 +341,5 @@ export default function stringWidth(input, options = {}) { return string.length; } - let width = 0; - const eastAsianWidthOptions = {ambiguousAsWide: !ambiguousIsNarrow}; - - for (const {segment} of segmenter.segment(string)) { - // Zero-width / non-printing clusters - if (isZeroWidthCluster(segment)) { - continue; - } - - // Emoji width logic - if (rgiEmojiRegex.test(segment) || isDoubleWidthNonRgiEmojiSequence(segment)) { - width += 2; - continue; - } - - const visibleSegment = baseVisible(segment); - const hangulWidth = hangulClusterWidth(visibleSegment, eastAsianWidthOptions); - if (hangulWidth !== undefined) { - width += hangulWidth; - continue; - } - - // Everything else: EAW of the cluster’s first visible scalar - const codePoint = visibleSegment.codePointAt(0); - width += eastAsianWidth(codePoint, eastAsianWidthOptions); - - // Add width for trailing spacing marks and Halfwidth/Fullwidth Forms (e.g., ﾞ, ﾟ, ｰ) - width += trailingWidth(visibleSegment, eastAsianWidthOptions); - } - - return width; + return textWidth(string, {ambiguousAsWide: !ambiguousIsNarrow}); } diff --git a/test.js b/test.js index c34131f..806a90a 100644 --- a/test.js +++ b/test.js @@ -1,5 +1,5 @@ import test from 'ava'; -import stringWidth from './index.js'; +import stringWidth, {_standaloneRanges as standaloneRanges} from './index.js'; const macro = test.macro((t, input, expected, options = {}) => { t.is(stringWidth(input, options), expected); @@ -337,3 +337,205 @@ test('ambiguous mixed with CJK (wide)', macro, '±你', 4, {ambiguousIsNarrow: f // `stripAnsi` guard: non-ANSI strings should not call `stripAnsi` test('non-ASCII without ANSI escapes', macro, '你好世界', 8); test('Latin1 without ANSI escapes', macro, 'résumé', 6); + +// ASCII runs are counted without segmenting; characters at the edges of a run may still join a cluster +test('long ASCII with trailing emoji', macro, 'a'.repeat(40) + '😀', 42); +test('long ASCII with leading emoji', macro, '😀' + 'a'.repeat(40), 42); +test('emoji between long ASCII runs', macro, 'a'.repeat(40) + '😀' + 'b'.repeat(40), 82); +test('emoji between short ASCII runs', macro, '😀 ok 😀', 8); +test('combining mark after long ASCII run', macro, 'a'.repeat(40) + '\u0301', 40); +test('combining mark between long ASCII runs', macro, 'a'.repeat(40) + '\u0301' + 'b'.repeat(40), 80); +test('ZWJ after long ASCII run', macro, 'a'.repeat(40) + '\u200D\u200D', 40); +test('prepend before long ASCII run', macro, '\u0600' + 'a'.repeat(40), 40); +test('keycap at the end of a long ASCII run', macro, 'a'.repeat(40) + '#\u20E3', 42); +test('keycap between long ASCII runs', macro, 'a'.repeat(40) + '#\uFE0F\u20E3' + 'b'.repeat(40), 82); +test('CJK between long ASCII runs', macro, 'a'.repeat(40) + '你好' + 'b'.repeat(40), 84); +test('ambiguous between long ASCII runs (wide)', macro, 'a'.repeat(40) + '±' + 'b'.repeat(40), 82, {ambiguousIsNarrow: false}); +test('tab between long ASCII runs', macro, 'a'.repeat(40) + '\t' + 'b'.repeat(40), 80); +test('ANSI codes around long ASCII with emoji', macro, '\u001B[32m' + 'a'.repeat(40) + ' ✅\u001B[0m', 43); + +// Standalone character runs: characters that never join a neighbouring cluster are counted without the segmenter, +// while the first and last character of a run stay with the segmenter +test('accented letter with combining mark', macro, 'é\u0301', 1); +test('accented word with combining mark', macro, 'café\u0301', 4); +test('long accented run with trailing combining mark', macro, 'é'.repeat(40) + '\u0301', 40); +test('long accented run with trailing combining mark (ambiguous wide)', macro, 'é'.repeat(40) + '\u0301', 80, {ambiguousIsNarrow: false}); +test('combining mark between accented runs', macro, 'é'.repeat(20) + '\u0301' + 'ñ'.repeat(20), 40); +test('combining mark between accented runs (ambiguous wide)', macro, 'é'.repeat(20) + '\u0301' + 'ñ'.repeat(20), 60, {ambiguousIsNarrow: false}); +test('CJK with VS16', macro, '你\uFE0F', 2); +test('CJK run with trailing VS15', macro, '你好世界'.repeat(5) + '\uFE0E', 40); +test('VS16 between CJK runs', macro, '你'.repeat(20) + '\uFE0F' + '好'.repeat(20), 80); +test('precomposed syllable with trailing jamo', macro, '가\u11A8', 2); +test('syllable run with trailing jamo', macro, '가'.repeat(20) + '\u11A8', 40); +test('leading jamo before syllable run', macro, '\u1100' + '가'.repeat(20), 42); +test('precomposed syllable with vowel jamo', macro, '가\u1161', 2); +test('emoji modifier after CJK', macro, '你\u{1F3FD}', 2); +test('emoji modifier between CJK runs', macro, '你'.repeat(20) + '\u{1F3FD}' + '好'.repeat(20), 80); +test('halfwidth kana run with voiced sound mark', macro, 'ｶ'.repeat(20) + '\uFF9E', 21); +test('hiragana run with combining dakuten', macro, 'か'.repeat(20) + '\u3099', 40); +test('Cyrillic run with combining mark', macro, 'ж'.repeat(20) + '\u0483', 20); +test('Cyrillic run with combining mark (ambiguous wide)', macro, 'ж'.repeat(20) + '\u0483', 40, {ambiguousIsNarrow: false}); +test('Greek run with combining mark', macro, 'α'.repeat(20) + '\u0345', 20); +test('copyright ZWJ copyright', macro, '©\u200D©', 2); +test('copyright run', macro, '©'.repeat(20), 20); +test('soft hyphen between Latin runs', macro, 'a'.repeat(20) + '\u00AD' + 'b'.repeat(20), 40); +test('Hangul filler between compatibility jamo runs', macro, 'ㄱ'.repeat(20) + '\u3164' + 'ㄴ'.repeat(20), 80); +test('CJK Extension B run', macro, '\u{20000}'.repeat(20), 40); +test('CJK Extension B run with combining mark', macro, '\u{20000}'.repeat(20) + '\u0301', 40); +test('combining mark before CJK Extension B run', macro, '\u0301' + '\u{20000}'.repeat(20), 40); +test('CJK Extension B run between combining marks', macro, '\u0301' + '\u{20000}'.repeat(20) + '\u0301', 40); +test('Hangul filler before CJK Extension B character', macro, '\u3164\u{2FA1F}', 2); +test('CJK Extension B character between symbols', macro, '\u2192\u{2A6D6}\u{1F3C1}\u4E3D', 7); +test('tag between symbol and CJK Extension G character', macro, '\u25FC\u{E0001}\u{3134A}', 3); +test('box drawing table row', macro, '│ 名前 │ OK │', 13); +test('box drawing table row (ambiguous wide)', macro, '│ 名前 │ OK │', 16, {ambiguousIsNarrow: false}); +test('ambiguous run (wide)', macro, '±'.repeat(20), 40, {ambiguousIsNarrow: false}); +test('mixed scripts prose', macro, 'Le café est déjà prêt 你好 Привет Γειά', 38); +test('mixed scripts prose (ambiguous wide)', macro, 'Le café est déjà prêt 你好 Привет Γειά', 51, {ambiguousIsNarrow: false}); +test('prepend before CJK run', macro, '\u0600' + '你'.repeat(20), 40); +test('CJK run with trailing ZWJ', macro, '你'.repeat(20) + '\u200D', 40); +test('flag after CJK run', macro, '你'.repeat(20) + '\u{1F1FA}\u{1F1F8}', 42); + +test('cached code point widths stay correct across cache eviction', t => { + // More distinct code points than the cache holds, checked twice so both cold and warm lookups are covered + for (let pass = 0; pass < 2; pass++) { + for (let codePoint = 0x4E_00; codePoint < 0x4E_00 + 3000; codePoint++) { + const character = String.fromCodePoint(codePoint); + t.is(stringWidth('\u0301' + character), 2); + t.is(stringWidth(character.repeat(3)), 6); + } + + for (let codePoint = 0x3_91; codePoint <= 0x3_A1; codePoint++) { + const character = String.fromCodePoint(codePoint); + t.is(stringWidth('\u0301' + character), 1); + t.is(stringWidth('\u0301' + character, {ambiguousIsNarrow: false}), 2); + t.is(stringWidth(character.repeat(3), {ambiguousIsNarrow: false}), 6); + } + } +}); + +test('standalone ranges contain only characters that never join a neighbouring cluster', t => { + // Anything that can take part in a join under UAX #29: Extend (marks, Grapheme_Extend, Emoji_Modifier), ZWJ and ZWNJ, + // SpacingMark (Mc), Control (Cc, Cf, Cs, Zl, Zp, default ignorables), Regional_Indicator, and the scripts that carry + // Prepend characters or Indic conjunct consonants and linkers. + const joiningProperties = [ + 'M', + 'Grapheme_Extend', + 'Emoji_Modifier', + 'Join_Control', + 'Cc', + 'Cf', + 'Cs', + 'Zl', + 'Zp', + 'Default_Ignorable_Code_Point', + 'Regional_Indicator', + 'Script=Arabic', + 'Script=Syriac', + 'Script=Devanagari', + 'Script=Bengali', + 'Script=Gurmukhi', + 'Script=Gujarati', + 'Script=Oriya', + 'Script=Tamil', + 'Script=Telugu', + 'Script=Kannada', + 'Script=Malayalam', + 'Script=Sinhala', + 'Script=Tibetan', + 'Script=Myanmar', + 'Script=Khmer', + 'Script=Thai', + 'Script=Lao', + 'Script=Balinese', + 'Script=Javanese', + ]; + const joiningRegex = new RegExp(`[${joiningProperties.map(property => `\\p{${property}}`).join('')}]`, 'u'); + + // Every script the ranges may contain; anything else (including Inherited, which holds combining marks) is a failure + const allowedScripts = ['Latin', 'Greek', 'Coptic', 'Cyrillic', 'Han', 'Hiragana', 'Katakana', 'Hangul', 'Bopomofo', 'Braille', 'Common']; + const allowedScriptRegex = new RegExp(`^[${allowedScripts.map(script => `\\p{Script=${script}}`).join('')}\\p{Unassigned}]$`, 'u'); + + // Grapheme_Cluster_Break=Prepend (Unicode 16) + const prependCodePoints = new Set([ + 0x6_00, + 0x6_01, + 0x6_02, + 0x6_03, + 0x6_04, + 0x6_05, + 0x6_DD, + 0x7_0F, + 0x8_90, + 0x8_91, + 0x8_E2, + 0xD_4E, + 0x1_10_BD, + 0x1_10_CD, + 0x1_11_C2, + 0x1_11_C3, + 0x1_19_3F, + 0x1_19_41, + 0x1_1A_3A, + 0x1_1A_84, + 0x1_1A_85, + 0x1_1A_86, + 0x1_1A_87, + 0x1_1A_88, + 0x1_1A_89, + 0x1_1D_46, + 0x1_1F_02, + ]); + + const isHangulJamo = codePoint => (codePoint >= 0x11_00 && codePoint <= 0x11_FF) || (codePoint >= 0xA9_60 && codePoint <= 0xA9_7F) || (codePoint >= 0xD7_B0 && codePoint <= 0xD7_FF); + + // The segmenter itself must agree: a standalone character never merges with a standalone neighbour of any kind + // (ASCII, Extended_Pictographic, precomposed Hangul, CJK, or another copy of itself) + const segmenter = new Intl.Segmenter(); + const probes = ['a', '©', '가', '你']; + + const failures = []; + for (const [start, end] of standaloneRanges) { + for (let codePoint = start; codePoint <= end; codePoint++) { + const character = String.fromCodePoint(codePoint); + const hex = codePoint.toString(16).toUpperCase(); + + if (joiningRegex.test(character) || !allowedScriptRegex.test(character) || prependCodePoints.has(codePoint) || isHangulJamo(codePoint)) { + failures.push(`U+${hex} can join a neighbouring cluster`); + continue; + } + + let probeString = character; + for (const probe of probes) { + probeString += probe + character; + } + + const segmentCount = [...segmenter.segment(probeString)].length; + if (segmentCount !== (probes.length * 2) + 1) { + failures.push(`U+${hex} merges with a neighbour (${segmentCount} segments)`); + } + } + } + + t.deepEqual(failures, []); +}); + +test('standalone ranges are sorted, disjoint and exclude the width-affecting halfwidth marks', t => { + let previousEnd = -1; + for (const [start, end] of standaloneRanges) { + t.true(start > previousEnd, `range starting at U+${start.toString(16)} overlaps or is out of order`); + t.true(end >= start); + previousEnd = end; + } + + const inRanges = codePoint => standaloneRanges.some(([start, end]) => codePoint >= start && codePoint <= end); + t.false(inRanges(0xFF_9E)); + t.false(inRanges(0xFF_9F)); + t.false(inRanges(0xAD)); + t.false(inRanges(0x3_00)); + t.false(inRanges(0x20_0D)); + t.false(inRanges(0x30_99)); + t.true(inRanges(0x4E_00)); + t.true(inRanges(0xAC_00)); + t.true(inRanges(0xE9)); +});