From 94faeaf8e1e521e3edef1434aa3917b42f5d8bd7 Mon Sep 17 00:00:00 2001 From: insjang Date: Wed, 2 Sep 2026 21:25:24 +0900 Subject: [PATCH 1/4] Terminal: count East Asian Wide characters as two columns The emulator assumed every character occupies one cell, so Hangul, Han and Kana text, fullwidth forms and emoji were placed one column short per character and the screen fell apart as soon as a program laid text out for a real terminal (line editors, curses UIs, Ink based CLIs). Add CharWidth, a UAX #11 East Asian Width lookup: Wide and Fullwidth count as two columns, combining marks and controls as zero, Ambiguous as one, as UAX #11 recommends outside an East Asian legacy context. The emulator advances the cursor by that width and stores a NUL filler in the second cell of a wide character, never splits one across the right margin, blanks the other half when either half is overwritten and counts insert mode in cells. The renderer skips the fillers so a fixed width font draws a wide glyph over both cells, falls back to placing each character at its own cell when the font does not advance exactly one cell per column, and draws a character beyond the BMP whole. A partial repaint that starts on the second cell of a wide character is widened to its first, and copying drops the fillers. Tests cover the width table, placement, the margin, overwriting halves and insert mode. --- .../META-INF/MANIFEST.MF | 2 +- .../emulator/VT100EmulatorBackend.java | 90 ++++++++++++- .../terminal/internal/model/CharWidth.java | 122 ++++++++++++++++++ .../textcanvas/AbstractTextCanvasModel.java | 14 +- .../internal/textcanvas/TextCanvas.java | 25 ++++ .../internal/textcanvas/TextLineRenderer.java | 73 ++++++++++- .../emulator/VT100EmulatorBackendTest.java | 89 +++++++++++++ .../terminal/internal/model/AllTestSuite.java | 1 + .../internal/model/CharWidthTest.java | 69 ++++++++++ 9 files changed, 473 insertions(+), 12 deletions(-) create mode 100644 terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java create mode 100644 terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/CharWidthTest.java diff --git a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF index ff5a3fb72be..0c618bbd154 100644 --- a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF +++ b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: %pluginName Bundle-SymbolicName: org.eclipse.terminal.control; singleton:=true -Bundle-Version: 1.1.200.qualifier +Bundle-Version: 1.1.300.qualifier Bundle-Activator: org.eclipse.terminal.internal.control.impl.TerminalPlugin Bundle-Vendor: %providerName Bundle-Localization: plugin diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java index 4dfc1f1b560..e4e13e270b5 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java @@ -14,6 +14,7 @@ *******************************************************************************/ package org.eclipse.terminal.internal.emulator; +import org.eclipse.terminal.internal.model.CharWidth; import org.eclipse.terminal.model.ITerminalTextData; import org.eclipse.terminal.model.TerminalStyle; @@ -311,7 +312,7 @@ public void appendString(String buffer) { synchronized (fTerminal) { char[] chars = buffer.toCharArray(); if (fInsertMode) { - insertCharacters(chars.length); + insertCharacters(CharWidth.ofString(buffer)); // room in cells, not characters } int line = toAbsoluteLine(fCursorLine); int i = 0; @@ -319,10 +320,50 @@ public void appendString(String buffer) { if (fWrapPending) { line = doLineWrap(); } - int n = Math.min(fColumns - fCursorColumn, chars.length - i); - fTerminal.setChars(line, fCursorColumn, chars, i, n, fStyle); - int col = fCursorColumn + n; - i += n; + int room = fColumns - fCursorColumn; + int col; + int n = narrowRun(chars, i, room); + if (n > 0) { + breakWideChar(line, fCursorColumn); + breakWideChar(line, fCursorColumn + n - 1); + fTerminal.setChars(line, fCursorColumn, chars, i, n, fStyle); + col = fCursorColumn + n; + i += n; + } else { + int codePoint = Character.codePointAt(chars, i); + int charsUsed = Character.charCount(codePoint); + int width = CharWidth.of(codePoint); + if (width == 0) { + // combining marks and other non-printing code points occupy no cell + i += charsUsed; + continue; + } + // a surrogate pair cannot share a cell, so it always takes two + if (charsUsed == 2) { + width = 2; + } + if (width > room) { + if (fCursorColumn > 0) { + // a wide character is never split across the right margin + line = doLineWrap(); + continue; + } + // terminal narrower than the character itself + width = room; + } + breakWideChar(line, fCursorColumn); + breakWideChar(line, fCursorColumn + width - 1); + if (charsUsed == 2) { + fTerminal.setChars(line, fCursorColumn, chars, i, 2, fStyle); + } else { + fTerminal.setChar(line, fCursorColumn, chars[i], fStyle); + if (width == 2) { + fTerminal.setChar(line, fCursorColumn + 1, '\000', fStyle); + } + } + col = fCursorColumn + width; + i += charsUsed; + } // wrap needed? if (col == fColumns) { if (fVT100LineWrapping) { @@ -339,6 +380,45 @@ public void appendString(String buffer) { } } + /** + * A wide character owns two cells. Overwriting either one leaves the other + * stranded: a filler with nothing in front of it, or a glyph that now spills + * over whatever was written next to it. Blanking the partner before the write + * goes in keeps the line honest, which is what a terminal is expected to do. + */ + private void breakWideChar(int line, int col) { + if (col < 0 || col >= fColumns) { + return; + } + char c = fTerminal.getChar(line, col); + if (c == '\000') { + if (col > 0 && CharWidth.of(fTerminal.getChar(line, col - 1)) == 2) { + blank(line, col - 1); + } + } else if (CharWidth.of(c) == 2 && col + 1 < fColumns && fTerminal.getChar(line, col + 1) == '\000') { + blank(line, col + 1); + } + } + + private void blank(int line, int col) { + fTerminal.setChar(line, col, ' ', fTerminal.getStyle(line, col)); + } + + /** + * Length of the run of characters starting at offset that each + * occupy exactly one cell, so that they can be copied in one block. Capped at + * max cells. Zero when the run does not start with such a + * character, which sends the caller down the code point by code point path. + */ + private static int narrowRun(char[] chars, int offset, int max) { + int n = 0; + while (n < max && offset + n < chars.length && !Character.isSurrogate(chars[offset + n]) + && CharWidth.of(chars[offset + n]) == 1) { + n++; + } + return n; + } + private int doLineWrap() { int line; line = toAbsoluteLine(fCursorLine); diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java new file mode 100644 index 00000000000..6966ba67a11 --- /dev/null +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java @@ -0,0 +1,122 @@ +/******************************************************************************* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.terminal.internal.model; + +/** + * Display width of a code point, following Unicode Standard Annex #11 + * (East Asian Width). Used by the emulator to keep its column arithmetic in + * step with what a terminal application assumes. + *

+ * East Asian Wide (W) and Fullwidth (F) count as two columns. Ambiguous (A) is + * treated as narrow, as UAX #11 recommends for a context with no East Asian + * legacy encoding. Combining marks and non-printing characters count as zero. + */ +public final class CharWidth { + + private CharWidth() { + } + + /** + * Wide and Fullwidth ranges from EastAsianWidth-17.0.0.txt, as inclusive + * [start, end] pairs in ascending order. Everything not listed here defaults + * to Narrow, which is what the file's {@code @missing} line specifies. + */ + private static final int[] WIDE_RANGES = { + 0x01100, 0x0115F, 0x0231A, 0x0231B, 0x02329, 0x0232A, 0x023E9, 0x023EC, + 0x023F0, 0x023F0, 0x023F3, 0x023F3, 0x025FD, 0x025FE, 0x02614, 0x02615, + 0x02630, 0x02637, 0x02648, 0x02653, 0x0267F, 0x0267F, 0x0268A, 0x0268F, + 0x02693, 0x02693, 0x026A1, 0x026A1, 0x026AA, 0x026AB, 0x026BD, 0x026BE, + 0x026C4, 0x026C5, 0x026CE, 0x026CE, 0x026D4, 0x026D4, 0x026EA, 0x026EA, + 0x026F2, 0x026F3, 0x026F5, 0x026F5, 0x026FA, 0x026FA, 0x026FD, 0x026FD, + 0x02705, 0x02705, 0x0270A, 0x0270B, 0x02728, 0x02728, 0x0274C, 0x0274C, + 0x0274E, 0x0274E, 0x02753, 0x02755, 0x02757, 0x02757, 0x02795, 0x02797, + 0x027B0, 0x027B0, 0x027BF, 0x027BF, 0x02B1B, 0x02B1C, 0x02B50, 0x02B50, + 0x02B55, 0x02B55, 0x02E80, 0x02E99, 0x02E9B, 0x02EF3, 0x02F00, 0x02FD5, + 0x02FF0, 0x0303E, 0x03041, 0x03096, 0x03099, 0x030FF, 0x03105, 0x0312F, + 0x03131, 0x0318E, 0x03190, 0x031E5, 0x031EF, 0x0321E, 0x03220, 0x03247, + 0x03250, 0x0A48C, 0x0A490, 0x0A4C6, 0x0A960, 0x0A97C, 0x0AC00, 0x0D7A3, + 0x0F900, 0x0FAFF, 0x0FE10, 0x0FE19, 0x0FE30, 0x0FE52, 0x0FE54, 0x0FE66, + 0x0FE68, 0x0FE6B, 0x0FF01, 0x0FF60, 0x0FFE0, 0x0FFE6, 0x16FE0, 0x16FE4, + 0x16FF0, 0x16FF6, 0x17000, 0x18CD5, 0x18CFF, 0x18D1E, 0x18D80, 0x18DF2, + 0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122, + 0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167, + 0x1B170, 0x1B2FB, 0x1D300, 0x1D356, 0x1D360, 0x1D376, 0x1F004, 0x1F004, + 0x1F0CF, 0x1F0CF, 0x1F18E, 0x1F18E, 0x1F191, 0x1F19A, 0x1F200, 0x1F202, + 0x1F210, 0x1F23B, 0x1F240, 0x1F248, 0x1F250, 0x1F251, 0x1F260, 0x1F265, + 0x1F300, 0x1F320, 0x1F32D, 0x1F335, 0x1F337, 0x1F37C, 0x1F37E, 0x1F393, + 0x1F3A0, 0x1F3CA, 0x1F3CF, 0x1F3D3, 0x1F3E0, 0x1F3F0, 0x1F3F4, 0x1F3F4, + 0x1F3F8, 0x1F43E, 0x1F440, 0x1F440, 0x1F442, 0x1F4FC, 0x1F4FF, 0x1F53D, + 0x1F54B, 0x1F54E, 0x1F550, 0x1F567, 0x1F57A, 0x1F57A, 0x1F595, 0x1F596, + 0x1F5A4, 0x1F5A4, 0x1F5FB, 0x1F64F, 0x1F680, 0x1F6C5, 0x1F6CC, 0x1F6CC, + 0x1F6D0, 0x1F6D2, 0x1F6D5, 0x1F6D8, 0x1F6DC, 0x1F6DF, 0x1F6EB, 0x1F6EC, + 0x1F6F4, 0x1F6FC, 0x1F7E0, 0x1F7EB, 0x1F7F0, 0x1F7F0, 0x1F90C, 0x1F93A, + 0x1F93C, 0x1F945, 0x1F947, 0x1F9FF, 0x1FA70, 0x1FA7C, 0x1FA80, 0x1FA8A, + 0x1FA8E, 0x1FAC6, 0x1FAC8, 0x1FAC8, 0x1FACD, 0x1FADC, 0x1FADF, 0x1FAEA, + 0x1FAEF, 0x1FAF8, 0x20000, 0x2FFFD, 0x30000, 0x3FFFD + }; + + /** @return 0 for combining and non-printing, 2 for East Asian W/F, else 1 */ + public static int of(int codePoint) { + if (codePoint < 0x0080) { + return codePoint < 0x20 || codePoint == 0x7F ? 0 : 1; + } + return isZeroWidth(codePoint) ? 0 : isWide(codePoint) ? 2 : 1; + } + + /** @return the total display width of {@code s} */ + public static int ofString(String s) { + return s.codePoints().map(CharWidth::of).sum(); + } + + /** + * A {@code '\000'} means one of two things in a line of cells: the filler that + * a wide character puts in the cell it also covers, which carries no text of + * its own, or a cell that was never written or has been erased, which reads as + * a space. + * + * @return whether the cell at {@code index} is the filler of the character + * before it + */ + public static boolean isFiller(CharSequence text, int index) { + return text.charAt(index) == '\000' && index > 0 && of(Character.codePointBefore(text, index)) == 2; + } + + private static boolean isZeroWidth(int codePoint) { + // Hangul conjoining jamo vowels and trailing consonants: EAW lists them as + // neutral, but they combine into the leading consonant before them. + if (codePoint >= 0x1160 && codePoint <= 0x11FF) { + return true; + } + switch (Character.getType(codePoint)) { + case Character.NON_SPACING_MARK: + case Character.ENCLOSING_MARK: + case Character.CONTROL: + case Character.FORMAT: + return true; + default: + return false; + } + } + + private static boolean isWide(int codePoint) { + int lo = 0, hi = WIDE_RANGES.length / 2 - 1; + while (lo <= hi) { + int mid = (lo + hi) >>> 1, i = mid * 2; + if (codePoint < WIDE_RANGES[i]) { + hi = mid - 1; + } else if (codePoint > WIDE_RANGES[i + 1]) + lo = mid + 1; + else { + return true; + } + } + return false; + } +} diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java index db57118ddc5..09acdc6d71a 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java @@ -21,6 +21,7 @@ import org.eclipse.core.runtime.Platform; import org.eclipse.swt.graphics.Point; import org.eclipse.terminal.connector.Logger; +import org.eclipse.terminal.internal.model.CharWidth; import org.eclipse.terminal.model.ITerminalTextDataReadOnly; import org.eclipse.terminal.model.ITerminalTextDataSnapshot; import org.eclipse.terminal.model.TextRange; @@ -427,8 +428,17 @@ private static String scrubLine(String text) { } text = text.substring(0, i + 1); // - // null means space - return text.replace('\000', ' '); + // null means space, unless it is the filler of a wide character + StringBuilder scrubbed = new StringBuilder(text.length()); + for (int j = 0; j < text.length(); j++) { + char c = text.charAt(j); + if (c != '\000') { + scrubbed.append(c); + } else if (!CharWidth.isFiller(text, j)) { + scrubbed.append(' '); + } + } + return scrubbed.toString(); } /** diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextCanvas.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextCanvas.java index 92f924be1a7..9d941c9099d 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextCanvas.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextCanvas.java @@ -43,6 +43,7 @@ import org.eclipse.swt.graphics.Rectangle; import org.eclipse.swt.widgets.Composite; import org.eclipse.terminal.control.ITerminalMouseListener; +import org.eclipse.terminal.internal.model.CharWidth; import org.eclipse.terminal.model.ITerminalTextDataReadOnly; import org.eclipse.terminal.model.TerminalColor; @@ -535,9 +536,33 @@ protected void repaintRange(int col, int line, int width, int height) { @Override protected void drawLine(GC gc, int line, int x, int y, int colFirst, int colLast) { + // A wide character spans two cells and has to be drawn from the first of + // them. When the damaged area starts on the second one, widen the range so + // the glyph is drawn from its own origin. Clipping then keeps only the half + // that was actually damaged, which is the half that needed repainting. + if (isFillerCell(line, colFirst)) { + colFirst--; + x -= getCellWidth(); + } + // Same at the other end: a wide character starting in the last cell of the + // range would be cut in half by the edge of the range. + if (isFillerCell(line, colLast)) { + colLast++; + } fCellRenderer.drawLine(fCellCanvasModel, gc, line, x, y, colFirst, colLast); } + /** + * @return whether the cell holds the second half of a wide character + */ + private boolean isFillerCell(int line, int col) { + ITerminalTextDataReadOnly text = fCellCanvasModel.getTerminalText(); + if (col <= 0 || col >= text.getWidth() || line < 0 || line >= text.getHeight()) { + return false; + } + return text.getChar(line, col) == '\000' && CharWidth.of(text.getChar(line, col - 1)) == 2; + } + @Override protected Color getTerminalBackgroundColor(Device device) { return fCellRenderer.getDefaultBackgroundColor(); diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextLineRenderer.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextLineRenderer.java index 8df74b82b22..a225b8486d2 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextLineRenderer.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/TextLineRenderer.java @@ -26,6 +26,7 @@ import org.eclipse.swt.graphics.GC; import org.eclipse.swt.graphics.Point; import org.eclipse.terminal.connector.Logger; +import org.eclipse.terminal.internal.model.CharWidth; import org.eclipse.terminal.model.ITerminalTextDataReadOnly; import org.eclipse.terminal.model.LineSegment; import org.eclipse.terminal.model.TerminalColor; @@ -159,19 +160,83 @@ private void drawText(GC gc, int x, int y, int colFirst, int col, String text) { // draw the background // TODO why does this not work??????? // gc.fillRectangle(x,y,fStyleMap.getFontWidth()*text.length(),fStyleMap.getFontHeight()); + int xx = x + offset; for (int i = 0; i < text.length(); i++) { char c = text.charAt(i); - int xx = x + offset + i * fStyleMap.getFontWidth(); + int cells = cellsAt(text, i); // TODO why do I have to draw the background character by character?????? - gc.fillRectangle(xx, y, fStyleMap.getFontWidth(), fStyleMap.getFontHeight()); + gc.fillRectangle(xx, y, cells * fStyleMap.getFontWidth(), fStyleMap.getFontHeight()); if (c != ' ' && c != '\000') { gc.drawString(String.valueOf(c), fStyleMap.getCharOffset(c) + xx, y, false); } + xx += cells * fStyleMap.getFontWidth(); } } else { - text = text.replace('\000', ' '); - gc.drawString(text, x + offset, y, false); + // One call keeps whatever the font does with the run, ligatures included, + // but only while it advances exactly one cell per column. A character the + // font does not have is drawn from somewhere else and rarely does, and + // then everything after it on the line sits in the wrong column. + String drawn = withoutFillers(text); + if (gc.textExtent(drawn).x == text.length() * getCellWidth()) { + gc.drawString(drawn, x + offset, y, false); + } else { + drawCellByCell(gc, x + offset, y, text); + } + } + } + + /** + * Puts every character at the start of its own cell, so the columns hold no + * matter what the font makes of it. A wide character is left to cover the cell + * of the filler that follows it. + */ + private void drawCellByCell(GC gc, int x, int y, String text) { + // The whole run at once, because the characters are drawn over it one at a + // time and the cells between them would otherwise keep what was there before. + gc.fillRectangle(x, y, text.length() * getCellWidth(), getCellHeight()); + for (int i = 0; i < text.length();) { + // A character beyond the BMP is two chars in two cells, and has to be + // drawn whole: half of a surrogate pair is no character at all. + int n = Character.charCount(text.codePointAt(i)); + char c = text.charAt(i); + if (c != ' ' && c != '\000') { + gc.drawString(text.substring(i, i + n), x + i * getCellWidth(), y, true); + } + i += n; + } + } + + /** + * Cells taken up by the character at index: none for the filler of + * a wide character, since the character it belongs to already covers it, two + * for a wide character, one for anything else. + */ + private static int cellsAt(String text, int index) { + if (CharWidth.isFiller(text, index)) { + return 0; + } + int codePoint = text.codePointAt(index); + // a surrogate pair takes two cells whatever its width, as it is stored + return Character.charCount(codePoint) == 2 || CharWidth.of(codePoint) == 2 ? 2 : 1; + } + + /** + * The text as it should be handed to a fixed width font: fillers dropped, since + * the wide character before them already spans their cell, and every other null + * turned into the space it stands for. What is left lines up column for column, + * as long as the font draws a wide character in exactly two cells. + */ + private static String withoutFillers(String text) { + StringBuilder drawn = new StringBuilder(text.length()); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c != '\000') { + drawn.append(c); + } else if (!CharWidth.isFiller(text, i)) { + drawn.append(' '); + } } + return drawn.toString(); } /** diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java index aecbc640616..c53ea81f425 100644 --- a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java @@ -1086,4 +1086,93 @@ public void testNewlineInTopAnchoredScrollRegionCurrentlyDiscardsTopLine() { assertNull(term.getChars(3)); assertEquals("4444", new String(term.getChars(4))); // footer below the region is untouched } + + @Test + public void testAppendStringWide() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(4); + vt100.setDimensions(4, 10); + vt100.setCursor(0, 0); + // a wide character takes two cells, the second holding a null filler + vt100.appendString("한글"); + assertEqualsTerm("한 글 \n" + " \n" + " \n" + " ", toMultiLineText(term)); + assertEquals(4, vt100.getCursorColumn()); + vt100.setCursor(1, 0); + vt100.appendString("a한b"); + assertEqualsTerm("한 글 \n" + "a한 b \n" + " \n" + " ", toMultiLineText(term)); + assertEquals(4, vt100.getCursorColumn()); + // a character beyond the BMP is two chars in two cells + vt100.setCursor(2, 0); + vt100.appendString("a😀b"); + assertEquals(4, vt100.getCursorColumn()); + assertEquals("a😀b", new String(term.getChars(2), 0, 4)); + // a combining mark takes no cell + vt100.setCursor(3, 0); + vt100.appendString("e\u0301x"); + assertEquals(2, vt100.getCursorColumn()); + } + + @Test + public void testAppendStringWideAtMargin() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(4); + vt100.setDimensions(4, 5); + vt100.setCursor(0, 0); + vt100.appendString("abc한"); + assertEqualsTerm("abc한 \n" + " \n" + " \n" + " ", toMultiLineText(term)); + // a wide character is never split across the margin: it goes to the next line whole + vt100.setCursor(1, 0); + vt100.appendString("abcd한"); + assertEqualsTerm("abc한 \n" + "abcd \n" + "한 \n" + " ", toMultiLineText(term)); + assertTrue(term.isWrappedLine(1)); + assertEquals(2, vt100.getCursorLine()); + assertEquals(2, vt100.getCursorColumn()); + } + + @Test + public void testOverwriteHalfOfWide() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(4); + vt100.setDimensions(4, 10); + // writing over the glyph leaves its filler blank, and over the filler leaves the glyph blank + vt100.setCursor(0, 0); + vt100.appendString("한글"); + vt100.setCursor(0, 0); + vt100.appendString("x"); + assertEquals("x 글\000", new String(term.getChars(0), 0, 4)); + vt100.setCursor(1, 0); + vt100.appendString("한글"); + vt100.setCursor(1, 1); + vt100.appendString("x"); + assertEquals(" x글\000", new String(term.getChars(1), 0, 4)); + vt100.setCursor(2, 0); + vt100.appendString("한글"); + vt100.setCursor(2, 3); + vt100.appendString("나"); + assertEquals("한\000 나\000", new String(term.getChars(2), 0, 5)); + // narrow over narrow is untouched by any of this + vt100.setCursor(3, 0); + vt100.appendString("abcd"); + vt100.setCursor(3, 1); + vt100.appendString("XY"); + assertEquals("aXYd", new String(term.getChars(3), 0, 4)); + } + + @Test + public void testInsertModeWide() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(1); + vt100.setDimensions(1, 10); + vt100.setCursor(0, 0); + vt100.appendString("abcdef"); + vt100.setCursorColumn(1); + vt100.setInsertMode(true); + vt100.appendString("한"); + // pushes the rest along by two cells, not one + assertEquals("a한\000bcdef", new String(term.getChars(0), 0, 8)); + } } diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AllTestSuite.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AllTestSuite.java index 47660269f0a..236b1b61dc8 100644 --- a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AllTestSuite.java +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AllTestSuite.java @@ -21,6 +21,7 @@ */ @Suite @SelectClasses({ // + CharWidthTest.class, // SnapshotChangesTest.class, // SynchronizedTerminalTextDataTest.class, // TerminalTextDataFastScrollTest.class, // diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/CharWidthTest.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/CharWidthTest.java new file mode 100644 index 00000000000..dfe94127a8d --- /dev/null +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/CharWidthTest.java @@ -0,0 +1,69 @@ +/******************************************************************************* + * Copyright (c) 2026 Contributors to the Eclipse Foundation. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License 2.0 + * which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.terminal.internal.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class CharWidthTest { + + @Test + public void testNarrow() { + assertEquals(1, CharWidth.of('A')); + assertEquals(1, CharWidth.of(0x00E9)); // e with acute + assertEquals(1, CharWidth.of(0x00B1)); // ambiguous: plus-minus + assertEquals(1, CharWidth.of(0x2500)); // ambiguous: box drawing + assertEquals(1, CharWidth.of(0xFFFD)); // ambiguous: replacement character + } + + @Test + public void testWide() { + assertEquals(2, CharWidth.of(0xAC00)); // hangul syllable + assertEquals(2, CharWidth.of(0x6F22)); // han ideograph + assertEquals(2, CharWidth.of(0xFF21)); // fullwidth A + assertEquals(2, CharWidth.of(0x1100)); // hangul jamo, leading consonant + assertEquals(2, CharWidth.of(0x3131)); // hangul compatibility jamo + assertEquals(2, CharWidth.of(0x3000)); // ideographic space + assertEquals(2, CharWidth.of(0x1F600)); // emoji + } + + @Test + public void testZeroWidth() { + assertEquals(0, CharWidth.of(0x0301)); // combining acute + assertEquals(0, CharWidth.of(0x200B)); // zero width space + assertEquals(0, CharWidth.of(0x1161)); // hangul jamo vowel, combines with the consonant before it + assertEquals(0, CharWidth.of('\n')); + assertEquals(0, CharWidth.of(0)); + assertEquals(0, CharWidth.of(0x7F)); + assertEquals(0, CharWidth.of(0x85)); // C1 control + } + + @Test + public void testOfString() { + assertEquals(0, CharWidth.ofString("")); + assertEquals(3, CharWidth.ofString("abc")); + assertEquals(7, CharWidth.ofString("한글abc")); // two hangul syllables, three letters + assertEquals(4, CharWidth.ofString("a😀b")); // surrogate pair counts once, as two cells + assertEquals(1, CharWidth.ofString("é")); // combining mark adds nothing + } + + @Test + public void testIsFiller() { + assertTrue(CharWidth.isFiller("가\000", 1)); + assertTrue(CharWidth.isFiller("😀\000", 2)); + assertFalse(CharWidth.isFiller("a\000", 1)); // an empty cell after a narrow character + assertFalse(CharWidth.isFiller("\000a", 0)); + assertFalse(CharWidth.isFiller("ab", 1)); + assertFalse(CharWidth.isFiller("가\000\000", 2)); // only the first null is the filler + } +} From bda043ccf7366913495d73a7f60bc1adb94f1b21 Mon Sep 17 00:00:00 2001 From: insjang Date: Wed, 2 Sep 2026 22:24:00 +0900 Subject: [PATCH 2/4] Terminal: compose combining marks, and keep a grapheme cluster in two cells A combining mark has no cell of its own. It was dropped, which lost data: an accent typed as a separate mark, a Hangul syllable sent as conjoining jamo, a Japanese voicing mark. A mark now composes with the character before it (NFC) whenever the two have a single form, which covers all of those. What has no composed form is a grapheme cluster: an emoji joined to others with a zero width joiner, one with a skin tone, a character with a presentation selector, a keycap, a flag made of two regional indicators. Windows Terminal (1.22+) and the libraries programs use to lay text out give such a cluster two cells however many characters it runs to, so the emulator has to count the same way or the columns drift. The cells keep the cluster's first character, which is what gets drawn; the whole cluster is kept beside the line and copied out whole, so what the user copies is what the program wrote. Writing either cell forgets the cluster, and it moves with its line when lines scroll or are copied. A narrow character asked to be shown as an emoji (VS16, as in a red heart) takes a second cell, as those terminals give it. Regional indicator pairs join into one flag. Marks that compose still compose first, so Hangul and accented Latin are unchanged. This builds on the East Asian Width change (CharWidth and the two-cell placement), and adds getCluster/setCluster to the text data API, hence the version bump. --- .../META-INF/MANIFEST.MF | 2 +- .../emulator/VT100EmulatorBackend.java | 145 +++++++++++++++++- .../model/SynchronizedTerminalTextData.java | 10 ++ .../internal/model/TerminalTextData.java | 11 ++ .../model/TerminalTextDataFastScroll.java | 12 ++ .../model/TerminalTextDataSnapshot.java | 5 + .../internal/model/TerminalTextDataStore.java | 52 +++++++ .../model/TerminalTextDataWindow.java | 12 ++ .../textcanvas/AbstractTextCanvasModel.java | 31 +++- .../terminal/model/ITerminalTextData.java | 8 + .../model/ITerminalTextDataReadOnly.java | 14 ++ .../emulator/VT100EmulatorBackendTest.java | 58 +++++++ .../model/AbstractITerminalTextDataTest.java | 32 ++++ .../model/TerminalTextDataWindowTest.java | 14 ++ 14 files changed, 389 insertions(+), 17 deletions(-) diff --git a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF index 0c618bbd154..2054f6b2dad 100644 --- a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF +++ b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: %pluginName Bundle-SymbolicName: org.eclipse.terminal.control; singleton:=true -Bundle-Version: 1.1.300.qualifier +Bundle-Version: 1.2.0.qualifier Bundle-Activator: org.eclipse.terminal.internal.control.impl.TerminalPlugin Bundle-Vendor: %providerName Bundle-Localization: plugin diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java index e4e13e270b5..3e4f3431305 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java @@ -14,6 +14,8 @@ *******************************************************************************/ package org.eclipse.terminal.internal.emulator; +import java.text.Normalizer; + import org.eclipse.terminal.internal.model.CharWidth; import org.eclipse.terminal.model.ITerminalTextData; import org.eclipse.terminal.model.TerminalStyle; @@ -317,6 +319,25 @@ public void appendString(String buffer) { int line = toAbsoluteLine(fCursorLine); int i = 0; while (i < chars.length) { + int codePoint = Character.codePointAt(chars, i); + int charsUsed = Character.charCount(codePoint); + int width = CharWidth.of(codePoint); + // What takes no cell of its own goes onto the character before it, even + // one on the last cell with a wrap pending: it belongs there, not on the + // next line. + if (joinsCluster(line, codePoint)) { + i += charsUsed; + continue; + } + if (width == 0) { + // composed into the character before it where the two have one form; + // otherwise kept with it as a cluster, as a keycap is + if (!combine(line, codePoint)) { + attachToCluster(line, codePoint); + } + i += charsUsed; + continue; + } if (fWrapPending) { line = doLineWrap(); } @@ -330,14 +351,6 @@ public void appendString(String buffer) { col = fCursorColumn + n; i += n; } else { - int codePoint = Character.codePointAt(chars, i); - int charsUsed = Character.charCount(codePoint); - int width = CharWidth.of(codePoint); - if (width == 0) { - // combining marks and other non-printing code points occupy no cell - i += charsUsed; - continue; - } // a surrogate pair cannot share a cell, so it always takes two if (charsUsed == 2) { width = 2; @@ -380,6 +393,122 @@ public void appendString(String buffer) { } } + private static final int ZWJ = 0x200D, VS15 = 0xFE0E, VS16 = 0xFE0F; + /** a zero width joiner was the last thing written: whatever comes next joins the cluster before it */ + private boolean fJoinPending; + + /** + * A grapheme cluster takes two cells however many characters it runs to, which + * is how Windows Terminal and the programs that lay text out for it count. What + * joins the cluster before the cursor and so takes no cells of its own: what + * follows a zero width joiner, a skin tone, a presentation selector, a mark. A + * presentation selector after a narrow character (a heart, a digit) makes the + * cluster wide first, taking the cell after it. The cells keep the first + * character to draw; the whole cluster is kept beside them for copying. + * + * @return whether the character was taken into a cluster + */ + private boolean joinsCluster(int line, int codePoint) { + boolean join = fJoinPending; + fJoinPending = false; + if (codePoint == ZWJ) { + fJoinPending = attachToCluster(line, codePoint); + return true; + } + if (isRegionalIndicator(codePoint) && !join) { + // two indicators make a flag, so the second joins the first + String before = clusterBefore(line); + join = before != null && before.codePointCount(0, before.length()) == 1 + && isRegionalIndicator(before.codePointAt(0)); + } + boolean modifier = codePoint == VS15 || codePoint == VS16 || (codePoint >= 0x1F3FB && codePoint <= 0x1F3FF); + if (!join && !modifier) { + return false; + } + if (attachToCluster(line, codePoint)) { + return true; + } + if (codePoint != VS16 || fWrapPending || fCursorColumn == 0 || fCursorColumn >= fColumns) { + return false; + } + // A narrow character asked to be shown as an emoji: it gets a second cell. + int col = fCursorColumn - 1; + char base = fTerminal.getChar(line, col); + if (base == 0 || base == ' ') { + return false; + } + breakWideChar(line, fCursorColumn); + fTerminal.setChar(line, fCursorColumn, '\000', fTerminal.getStyle(line, col)); + fTerminal.setCluster(line, col, base + new String(Character.toChars(VS16))); + setCursorColumn(fCursorColumn + 1); + return true; + } + + private static boolean isRegionalIndicator(int codePoint) { + return codePoint >= 0x1F1E6 && codePoint <= 0x1F1FF; + } + + /** @return whether there was a two-cell cluster before the cursor for the character to go into */ + private boolean attachToCluster(int line, int codePoint) { + String cluster = clusterBefore(line); + if (cluster == null) { + return false; + } + fTerminal.setCluster(line, endColumn() - 2, cluster + new String(Character.toChars(codePoint))); + return true; + } + + /** Where the next character goes: past the margin while a wrap is pending, the cursor being held on the last cell. */ + private int endColumn() { + return fWrapPending ? fColumns : fCursorColumn; + } + + /** The cluster in the two cells before the cursor, or null when there is no two-cell character there. */ + private String clusterBefore(int line) { + int col = endColumn() - 2; + if (col < 0) { + return null; + } + String cluster = fTerminal.getCluster(line, col); + if (cluster != null) { + return cluster; + } + char first = fTerminal.getChar(line, col), second = fTerminal.getChar(line, col + 1); + if (Character.isHighSurrogate(first) && Character.isLowSurrogate(second)) { + return new String(new char[] { first, second }); + } + if (CharWidth.of(first) == 2 && second == '\000') { + return String.valueOf(first); + } + return null; + } + + /** + * A mark owns no cell of its own, so it has to go onto the character it followed + * or be lost. A cell holds one character, which is enough whenever the two have + * a single composed form - the accents, the Hangul jamo, the Japanese voicing + * marks. What has no such form is still dropped, there being nowhere to put it. + */ + private boolean combine(int line, int mark) { + int col = endColumn() - 1; + if (col > 0 && fTerminal.getChar(line, col) == '\000') { + col--; // the mark follows a wide character, and that is the cell it lives in + } + if (col < 0) { + return false; + } + char base = fTerminal.getChar(line, col); + if (base == 0 || base == ' ') { + return false; + } + String composed = Normalizer.normalize(base + new String(Character.toChars(mark)), Normalizer.Form.NFC); + if (composed.length() != 1) { + return false; + } + fTerminal.setChar(line, col, composed.charAt(0), fTerminal.getStyle(line, col)); + return true; + } + /** * A wide character owns two cells. Overwriting either one leaves the other * stranded: a filler with nothing in front of it, or a glyph that now spills diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/SynchronizedTerminalTextData.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/SynchronizedTerminalTextData.java index a142ef8578a..c94bee17ec9 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/SynchronizedTerminalTextData.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/SynchronizedTerminalTextData.java @@ -157,4 +157,14 @@ synchronized public boolean isWrappedLine(int line) { synchronized public void setWrappedLine(int line) { fData.setWrappedLine(line); } + + @Override + synchronized public String getCluster(int line, int column) { + return fData.getCluster(line, column); + } + + @Override + synchronized public void setCluster(int line, int column, String cluster) { + fData.setCluster(line, column, cluster); + } } diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextData.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextData.java index baa3b620df8..59402c5f04b 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextData.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextData.java @@ -334,4 +334,15 @@ public boolean isWrappedLine(int line) { public void setWrappedLine(int line) { fData.setWrappedLine(line); } + + @Override + public String getCluster(int line, int column) { + return fData.getCluster(line, column); + } + + @Override + public void setCluster(int line, int column, String cluster) { + fData.setCluster(line, column, cluster); + sendLineChangedToSnapshots(line); + } } diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataFastScroll.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataFastScroll.java index c76fab20863..9c848fe5da9 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataFastScroll.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataFastScroll.java @@ -302,6 +302,18 @@ private void validateLineParameter(int line) { } } + @Override + public String getCluster(int line, int column) { + validateLineParameter(line); + return fData.getCluster(getPositionOfLine(line), column); + } + + @Override + public void setCluster(int line, int column, String cluster) { + validateLineParameter(line); + fData.setCluster(getPositionOfLine(line), column, cluster); + } + @Override public void setWrappedLine(int line) { validateLineParameter(line); diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataSnapshot.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataSnapshot.java index 2a1688523eb..0d5cb7cafb9 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataSnapshot.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataSnapshot.java @@ -318,4 +318,9 @@ public ITerminalTextData getTerminalTextData() { public boolean isWrappedLine(int line) { return fSnapshot.isWrappedLine(line); } + + @Override + public String getCluster(int line, int column) { + return fSnapshot.getCluster(line, column); + } } diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java index 04763621546..e793e26cf9b 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java @@ -14,7 +14,9 @@ import java.lang.reflect.Array; import java.util.ArrayList; import java.util.BitSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.eclipse.terminal.model.ITerminalTextData; import org.eclipse.terminal.model.ITerminalTextDataSnapshot; @@ -34,10 +36,13 @@ public class TerminalTextDataStore implements ITerminalTextData { private int fCursorColumn; private int fCursorLine; final private BitSet fWrappedLines = new BitSet(); + /** per line, the clusters on it by first column; sparse, most lines have none */ + private Map[] fClusters; public TerminalTextDataStore() { fChars = new char[0][]; fStyle = new TerminalStyle[0][]; + fClusters = newClusters(0); fWidth = 0; } @@ -67,6 +72,7 @@ public void setDimensions(int height, int width) { } fStyle = (TerminalStyle[][]) resizeArray(fStyle, height); fChars = (char[][]) resizeArray(fChars, height); + fClusters = (Map[]) resizeArray(fClusters, height); } // clean the new lines if (height > fHeight) { @@ -192,6 +198,7 @@ public void setChar(int line, int column, char c, TerminalStyle style) { ensureLineLength(line, column + 1); fChars[line][column] = c; fStyle[line][column] = style; + forgetClusters(line, column, 1); } @Override @@ -210,6 +217,37 @@ public void setChars(int line, int column, char[] chars, int start, int len, Ter fChars[line][column + i] = chars[i + start]; fStyle[line][column + i] = style; } + forgetClusters(line, column, len); + } + + /** Writing a cell ends any cluster that had it, whether as its first cell or its second. */ + private void forgetClusters(int line, int column, int len) { + Map clusters = fClusters[line]; + if (clusters == null || clusters.isEmpty()) { + return; + } + for (int col = column - 1; col < column + len; col++) { + clusters.remove(col); + } + } + + @Override + public String getCluster(int line, int column) { + Map clusters = fClusters[line]; + return clusters == null ? null : clusters.get(column); + } + + @Override + public void setCluster(int line, int column, String cluster) { + if (fClusters[line] == null) { + fClusters[line] = new HashMap<>(); + } + fClusters[line].put(column, cluster); + } + + @SuppressWarnings("unchecked") + private static Map[] newClusters(int n) { + return new Map[n]; } @Override @@ -224,6 +262,7 @@ public void scroll(int startLine, int size, int shift) { for (int i = startLine; i < startLine + size + shift; i++) { fChars[i] = fChars[i - shift]; fStyle[i] = fStyle[i - shift]; + fClusters[i] = fClusters[i - shift]; fWrappedLines.set(i, fWrappedLines.get(i - shift)); } // then clean the opened lines @@ -233,6 +272,7 @@ public void scroll(int startLine, int size, int shift) { for (int i = startLine + size - 1; i >= startLine && i - shift >= 0; i--) { fChars[i] = fChars[i - shift]; fStyle[i] = fStyle[i - shift]; + fClusters[i] = fClusters[i - shift]; fWrappedLines.set(i, fWrappedLines.get(i - shift)); } cleanLines(startLine, Math.min(shift, getHeight() - startLine)); @@ -289,6 +329,7 @@ public void copy(ITerminalTextData source) { if (getHeight() != n) { fChars = new char[n][]; fStyle = new TerminalStyle[n][]; + fClusters = newClusters(n); } for (int i = 0; i < n; i++) { copyLine(source, i, i); @@ -324,6 +365,15 @@ public void copyLine(ITerminalTextData source, int sourceLine, int destLine) { fChars[destLine] = source.getChars(sourceLine); fStyle[destLine] = source.getStyles(sourceLine); fWrappedLines.set(destLine, source.isWrappedLine(sourceLine)); + fClusters[destLine] = null; + if (fChars[destLine] != null) { + for (int col = 0; col < fChars[destLine].length; col++) { + String cluster = source.getCluster(sourceLine, col); + if (cluster != null) { + setCluster(destLine, col, cluster); + } + } + } } @Override @@ -346,6 +396,7 @@ public void setLine(int line, char[] chars, TerminalStyle[] styles) { fChars[line] = chars.clone(); fStyle[line] = styles.clone(); fWrappedLines.clear(line); + fClusters[line] = null; } @Override @@ -363,6 +414,7 @@ public void cleanLine(int line) { fChars[line] = null; fStyle[line] = null; fWrappedLines.clear(line); + fClusters[line] = null; } @Override diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataWindow.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataWindow.java index 21e343ee6b8..a0f62f8350e 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataWindow.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataWindow.java @@ -261,6 +261,18 @@ public boolean isWrappedLine(int line) { return false; } + @Override + public String getCluster(int line, int column) { + return isInWindow(line) ? fData.getCluster(line - fWindowStartLine, column) : null; + } + + @Override + public void setCluster(int line, int column, String cluster) { + if (isInWindow(line)) { + fData.setCluster(line - fWindowStartLine, column, cluster); + } + } + @Override public void setWrappedLine(int line) { if (isInWindow(line)) { diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java index 09acdc6d71a..eb1084db612 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/textcanvas/AbstractTextCanvasModel.java @@ -441,6 +441,24 @@ private static String scrubLine(String text) { return scrubbed.toString(); } + /** + * The cells from from to to as text, with each two-cell + * cluster given in full in place of the one character its cells hold. + */ + private static String withClusters(ITerminalTextDataReadOnly data, int line, char[] chars, int from, int to) { + StringBuilder text = new StringBuilder(to - from); + for (int col = from; col < to; col++) { + String cluster = col + 1 < to ? data.getCluster(line, col) : null; + if (cluster != null) { + text.append(cluster); + col++; + } else { + text.append(chars[col]); + } + } + return text.toString(); + } + /** * Calculates the currently selected text * @return the currently selected text @@ -454,14 +472,11 @@ private String extractSelectedText() { String text; char[] chars = fSelectionSnapshot.getChars(line); if (chars != null) { - text = new String(chars); - if (line == fSeletionEndLine && fSelectionEndColumn >= 0) { - text = text.substring(0, Math.min(fSelectionEndColumn + 1, text.length())); - } - if (line == fSelectionStartLine) { - text = text.substring(Math.min(fSelectionStartCoumn, text.length())); - } - text = scrubLine(text); + int from = line == fSelectionStartLine ? Math.min(fSelectionStartCoumn, chars.length) : 0; + int to = line == fSeletionEndLine && fSelectionEndColumn >= 0 + ? Math.min(fSelectionEndColumn + 1, chars.length) + : chars.length; + text = scrubLine(withClusters(fSelectionSnapshot, line, chars, from, to)); } else { text = ""; //$NON-NLS-1$ } diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextData.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextData.java index f972bbbe28b..df6d729655b 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextData.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextData.java @@ -156,4 +156,12 @@ public interface ITerminalTextData extends ITerminalTextDataReadOnly { */ void setWrappedLine(int line); + /** + * Records the whole of a grapheme cluster drawn in the two cells from column, + * see {@link #getCluster(int, int)}. Writing either cell forgets it. + * @since 1.2 + */ + default void setCluster(int line, int column, String cluster) { + } + } diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextDataReadOnly.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextDataReadOnly.java index 9809e17ef37..81f112d662c 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextDataReadOnly.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/model/ITerminalTextDataReadOnly.java @@ -90,4 +90,18 @@ public interface ITerminalTextDataReadOnly { */ boolean isWrappedLine(int line); + /** + * A grapheme cluster is drawn in two cells however many characters make it up: + * an emoji joined to others, or given a skin tone or a presentation selector. + * The cells hold its first character; this is the whole of it. + * + * @param line + * @param column the first of the two cells + * @return the cluster, or null where the cells hold nothing more than they show + * @since 1.2 + */ + default String getCluster(int line, int column) { + return null; + } + } diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java index c53ea81f425..49bae9b59da 100644 --- a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackendTest.java @@ -1175,4 +1175,62 @@ public void testInsertModeWide() { // pushes the rest along by two cells, not one assertEquals("a한\000bcdef", new String(term.getChars(0), 0, 8)); } + + @Test + public void testCombiningMarks() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(4); + vt100.setDimensions(4, 10); + vt100.setCursor(0, 0); + // a mark composes with the character before it where the two have one form + vt100.appendString("e\u0301x"); + assertEquals("\u00e9x", new String(term.getChars(0), 0, 2)); + assertEquals(2, vt100.getCursorColumn()); + // Hangul conjoining jamo make a syllable, in one wide cell + vt100.setCursor(1, 0); + vt100.appendString("\u1112\u1161\u11ab|"); + assertEquals("\ud55c\000|", new String(term.getChars(1), 0, 3)); + // a mark with no composed form is kept with the character as a cluster + vt100.setCursor(2, 0); + vt100.appendString("1\ufe0f\u20e3|"); // keycap one + assertEquals("1\ufe0f\u20e3", term.getCluster(2, 0)); + assertEquals('|', term.getChar(2, 2)); + } + + @Test + public void testGraphemeClusters() { + ITerminalTextData term = makeITerminalTextData(); + IVT100EmulatorBackend vt100 = makeBakend(term); + term.setMaxHeight(4); + vt100.setDimensions(4, 20); + String family = "\ud83d\udc68\u200d\ud83d\udc69\u200d\ud83d\udc67"; // man ZWJ woman ZWJ girl + String heart = "\u2764\ufe0f"; // heavy black heart + emoji presentation + String thumbs = "\ud83d\udc4d\ud83c\udffd"; // thumbs up + medium skin tone + String flag = "\ud83c\uddf0\ud83c\uddf7"; // regional indicators K R + // a cluster takes two cells however many characters it runs to + vt100.setCursor(0, 0); + vt100.appendString(family + "|"); + assertEquals(3, vt100.getCursorColumn()); + assertEquals("\ud83d\udc68", new String(term.getChars(0), 0, 2)); // the cells hold the first of them + assertEquals(family, term.getCluster(0, 0)); // the whole is kept beside them + // a narrow character shown as an emoji takes a second cell + vt100.setCursor(1, 0); + vt100.appendString(heart + "|" + thumbs + "|" + flag + "|"); + assertEquals(9, vt100.getCursorColumn()); + assertEquals(heart, term.getCluster(1, 0)); + assertEquals(thumbs, term.getCluster(1, 3)); + assertEquals(flag, term.getCluster(1, 6)); + // writing over a cell forgets the cluster that had it + vt100.setCursor(1, 1); + vt100.appendString("x"); + assertNull(term.getCluster(1, 0)); + // a cluster on the last two cells, with the wrap pending, is still joined + vt100.setVT100LineWrapping(true); + vt100.setCursor(2, 18); + vt100.appendString(family); + assertEquals(family, term.getCluster(2, 18)); + vt100.appendString("y"); + assertEquals('y', term.getChar(3, 0)); + } } diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AbstractITerminalTextDataTest.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AbstractITerminalTextDataTest.java index f1d3857ba27..c7e0eac8c78 100644 --- a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AbstractITerminalTextDataTest.java +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/AbstractITerminalTextDataTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -655,4 +656,35 @@ public void testWrappedLines() { term.cleanLine(0); assertFalse(term.isWrappedLine(0)); } + + @Test + public void testClusters() { + ITerminalTextData term = makeITerminalTextData(); + term.setDimensions(4, 6); + term.setMaxHeight(4); + fill(term, "abcdef\nghijkl\nmnopqr\nstuvwx"); + assertNull(term.getCluster(1, 2)); + term.setCluster(1, 2, "ij\u200d"); + assertEquals("ij\u200d", term.getCluster(1, 2)); + // writing either of its cells forgets a cluster + term.setChar(1, 3, 'X', null); + assertNull(term.getCluster(1, 2)); + term.setCluster(1, 2, "ij\u200d"); + term.setChars(1, 0, new char[] { 'y', 'z' }, null); + assertEquals("ij\u200d", term.getCluster(1, 2)); // untouched cells keep theirs + term.setChars(1, 1, new char[] { 'y', 'z' }, null); + assertNull(term.getCluster(1, 2)); + // a cluster goes with its line when lines scroll, and is gone with a cleaned line + term.setCluster(2, 0, "mn\u200d"); + term.scroll(0, 4, -1); + assertEquals("mn\u200d", term.getCluster(1, 0)); + assertNull(term.getCluster(2, 0)); + term.cleanLine(1); + assertNull(term.getCluster(1, 0)); + // and is copied with its line + term.setCluster(0, 4, "ef\u200d"); + ITerminalTextData copy = makeITerminalTextData(); + copy.copy(term); + assertEquals("ef\u200d", copy.getCluster(0, 4)); + } } diff --git a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/TerminalTextDataWindowTest.java b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/TerminalTextDataWindowTest.java index 163af3be430..5d2f1dd3747 100644 --- a/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/TerminalTextDataWindowTest.java +++ b/terminal/tests/org.eclipse.terminal.test/src/org/eclipse/terminal/internal/model/TerminalTextDataWindowTest.java @@ -15,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -442,4 +443,17 @@ public void testWrappedLines() { term.cleanLine(3); assertFalse(term.isWrappedLine(3)); } + + @Override + @Test + public void testClusters() { + ITerminalTextData term = makeITerminalTextData(); + term.setDimensions(4, 6); + term.setCluster(0, 2, "ab\u200d"); // outside window + term.setCluster(3, 2, "ij\u200d"); + assertNull(term.getCluster(0, 2)); + assertEquals("ij\u200d", term.getCluster(3, 2)); + term.setChar(3, 3, 'X', null); + assertNull(term.getCluster(3, 2)); + } } From 33e5b9992eab0535377d7e8c1fc9abf92c8197bc Mon Sep 17 00:00:00 2001 From: insjang Date: Thu, 3 Sep 2026 12:33:31 +0900 Subject: [PATCH 3/4] Terminal: bump org.eclipse.terminal.model package export to 1.1.0 New @since 1.2 API was added to the package without bumping its Export-Package version, which API Tools flags as an error. --- .../bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF index 2054f6b2dad..8ef0e9347c0 100644 --- a/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF +++ b/terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF @@ -30,5 +30,5 @@ Export-Package: org.eclipse.terminal.connector;version="1.0.100"; org.eclipse.terminal.internal.model;x-internal:=true, org.eclipse.terminal.internal.preferences;x-internal:=true;x-friends:="org.eclipse.terminal.view.ui", org.eclipse.terminal.internal.textcanvas;x-internal:=true, - org.eclipse.terminal.model;version="1.0.100";uses:="org.eclipse.swt.graphics" + org.eclipse.terminal.model;version="1.1.0";uses:="org.eclipse.swt.graphics" Automatic-Module-Name: org.eclipse.terminal.control From 66aa868a938f0e8f22262fe946d187ad33226416 Mon Sep 17 00:00:00 2001 From: insjang Date: Thu, 3 Sep 2026 17:02:01 +0900 Subject: [PATCH 4/4] Terminal: suppress unchecked-cast warning on generic cluster array resize Matches the existing @SuppressWarnings("unchecked") convention used for the same generic array pattern in newClusters(). --- .../eclipse/terminal/internal/model/TerminalTextDataStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java index e793e26cf9b..91960b1e611 100644 --- a/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java +++ b/terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/TerminalTextDataStore.java @@ -57,6 +57,7 @@ public int getHeight() { } @Override + @SuppressWarnings("unchecked") public void setDimensions(int height, int width) { if (height < 0) { throw new IllegalArgumentException("Parameter 'height' can't be negative value:" + height); //$NON-NLS-1$