Terminal: compose combining marks, and keep a grapheme cluster in two cells

terminal/bundles/org.eclipse.terminal.control/META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: %pluginName
 Bundle-SymbolicName: org.eclipse.terminal.control; singleton:=true
-Bundle-Version: 1.1.200.qualifier
+Bundle-Version: 1.2.0.qualifier
 Bundle-Activator: org.eclipse.terminal.internal.control.impl.TerminalPlugin
 Bundle-Vendor: %providerName
 Bundle-Localization: plugin
@@ -30,5 +30,5 @@ Export-Package: org.eclipse.terminal.connector;version="1.0.100";
  org.eclipse.terminal.internal.model;x-internal:=true,
  org.eclipse.terminal.internal.preferences;x-internal:=true;x-friends:="org.eclipse.terminal.view.ui",
  org.eclipse.terminal.internal.textcanvas;x-internal:=true,
- org.eclipse.terminal.model;version="1.0.100";uses:="org.eclipse.swt.graphics"
+ org.eclipse.terminal.model;version="1.1.0";uses:="org.eclipse.swt.graphics"
 Automatic-Module-Name: org.eclipse.terminal.control

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/emulator/VT100EmulatorBackend.java

@@ -14,6 +14,9 @@
  *******************************************************************************/
 package org.eclipse.terminal.internal.emulator;
 
+import java.text.Normalizer;
+
+import org.eclipse.terminal.internal.model.CharWidth;
 import org.eclipse.terminal.model.ITerminalTextData;
 import org.eclipse.terminal.model.TerminalStyle;
 
@@ -311,18 +314,69 @@ public void appendString(String buffer) {
 		synchronized (fTerminal) {
 			char[] chars = buffer.toCharArray();
 			if (fInsertMode) {
-				insertCharacters(chars.length);
+				insertCharacters(CharWidth.ofString(buffer)); // room in cells, not characters
 			}
 			int line = toAbsoluteLine(fCursorLine);
 			int i = 0;
 			while (i < chars.length) {
+				int codePoint = Character.codePointAt(chars, i);
+				int charsUsed = Character.charCount(codePoint);
+				int width = CharWidth.of(codePoint);
+				// What takes no cell of its own goes onto the character before it, even
+				// one on the last cell with a wrap pending: it belongs there, not on the
+				// next line.
+				if (joinsCluster(line, codePoint)) {
+					i += charsUsed;
+					continue;
+				}
+				if (width == 0) {
+					// composed into the character before it where the two have one form;
+					// otherwise kept with it as a cluster, as a keycap is
+					if (!combine(line, codePoint)) {
+						attachToCluster(line, codePoint);
+					}
+					i += charsUsed;
+					continue;
+				}
 				if (fWrapPending) {
 					line = doLineWrap();
 				}
-				int n = Math.min(fColumns - fCursorColumn, chars.length - i);
-				fTerminal.setChars(line, fCursorColumn, chars, i, n, fStyle);
-				int col = fCursorColumn + n;
-				i += n;
+				int room = fColumns - fCursorColumn;
+				int col;
+				int n = narrowRun(chars, i, room);
+				if (n > 0) {
+					breakWideChar(line, fCursorColumn);
+					breakWideChar(line, fCursorColumn + n - 1);
+					fTerminal.setChars(line, fCursorColumn, chars, i, n, fStyle);
+					col = fCursorColumn + n;
+					i += n;
+				} else {
+					// a surrogate pair cannot share a cell, so it always takes two
+					if (charsUsed == 2) {
+						width = 2;
+					}
+					if (width > room) {
+						if (fCursorColumn > 0) {
+							// a wide character is never split across the right margin
+							line = doLineWrap();
+							continue;
+						}
+						// terminal narrower than the character itself
+						width = room;
+					}
+					breakWideChar(line, fCursorColumn);
+					breakWideChar(line, fCursorColumn + width - 1);
+					if (charsUsed == 2) {
+						fTerminal.setChars(line, fCursorColumn, chars, i, 2, fStyle);
+					} else {
+						fTerminal.setChar(line, fCursorColumn, chars[i], fStyle);
+						if (width == 2) {
+							fTerminal.setChar(line, fCursorColumn + 1, '\000', fStyle);
+						}
+					}
+					col = fCursorColumn + width;
+					i += charsUsed;
+				}
 				// wrap needed?
 				if (col == fColumns) {
 					if (fVT100LineWrapping) {
@@ -339,6 +393,161 @@ public void appendString(String buffer) {
 		}
 	}
 
+	private static final int ZWJ = 0x200D, VS15 = 0xFE0E, VS16 = 0xFE0F;
+	/** a zero width joiner was the last thing written: whatever comes next joins the cluster before it */
+	private boolean fJoinPending;
+
+	/**
+	 * A grapheme cluster takes two cells however many characters it runs to, which
+	 * is how Windows Terminal and the programs that lay text out for it count. What
+	 * joins the cluster before the cursor and so takes no cells of its own: what
+	 * follows a zero width joiner, a skin tone, a presentation selector, a mark. A
+	 * presentation selector after a narrow character (a heart, a digit) makes the
+	 * cluster wide first, taking the cell after it. The cells keep the first
+	 * character to draw; the whole cluster is kept beside them for copying.
+	 *
+	 * @return whether the character was taken into a cluster
+	 */
+	private boolean joinsCluster(int line, int codePoint) {
+		boolean join = fJoinPending;
+		fJoinPending = false;
+		if (codePoint == ZWJ) {
+			fJoinPending = attachToCluster(line, codePoint);
+			return true;
+		}
+		if (isRegionalIndicator(codePoint) && !join) {
+			// two indicators make a flag, so the second joins the first
+			String before = clusterBefore(line);
+			join = before != null && before.codePointCount(0, before.length()) == 1
+					&& isRegionalIndicator(before.codePointAt(0));
+		}
+		boolean modifier = codePoint == VS15 || codePoint == VS16 || (codePoint >= 0x1F3FB && codePoint <= 0x1F3FF);
+		if (!join && !modifier) {
+			return false;
+		}
+		if (attachToCluster(line, codePoint)) {
+			return true;
+		}
+		if (codePoint != VS16 || fWrapPending || fCursorColumn == 0 || fCursorColumn >= fColumns) {
+			return false;
+		}
+		// A narrow character asked to be shown as an emoji: it gets a second cell.
+		int col = fCursorColumn - 1;
+		char base = fTerminal.getChar(line, col);
+		if (base == 0 || base == ' ') {
+			return false;
+		}
+		breakWideChar(line, fCursorColumn);
+		fTerminal.setChar(line, fCursorColumn, '\000', fTerminal.getStyle(line, col));
+		fTerminal.setCluster(line, col, base + new String(Character.toChars(VS16)));
+		setCursorColumn(fCursorColumn + 1);
+		return true;
+	}
+
+	private static boolean isRegionalIndicator(int codePoint) {
+		return codePoint >= 0x1F1E6 && codePoint <= 0x1F1FF;
+	}
+
+	/** @return whether there was a two-cell cluster before the cursor for the character to go into */
+	private boolean attachToCluster(int line, int codePoint) {
+		String cluster = clusterBefore(line);
+		if (cluster == null) {
+			return false;
+		}
+		fTerminal.setCluster(line, endColumn() - 2, cluster + new String(Character.toChars(codePoint)));
+		return true;
+	}
+
+	/** Where the next character goes: past the margin while a wrap is pending, the cursor being held on the last cell. */
+	private int endColumn() {
+		return fWrapPending ? fColumns : fCursorColumn;
+	}
+
+	/** The cluster in the two cells before the cursor, or null when there is no two-cell character there. */
+	private String clusterBefore(int line) {
+		int col = endColumn() - 2;
+		if (col < 0) {
+			return null;
+		}
+		String cluster = fTerminal.getCluster(line, col);
+		if (cluster != null) {
+			return cluster;
+		}
+		char first = fTerminal.getChar(line, col), second = fTerminal.getChar(line, col + 1);
+		if (Character.isHighSurrogate(first) && Character.isLowSurrogate(second)) {
+			return new String(new char[] { first, second });
+		}
+		if (CharWidth.of(first) == 2 && second == '\000') {
+			return String.valueOf(first);
+		}
+		return null;
+	}
+
+	/**
+	 * A mark owns no cell of its own, so it has to go onto the character it followed
+	 * or be lost. A cell holds one character, which is enough whenever the two have
+	 * a single composed form - the accents, the Hangul jamo, the Japanese voicing
+	 * marks. What has no such form is still dropped, there being nowhere to put it.
+	 */
+	private boolean combine(int line, int mark) {
+		int col = endColumn() - 1;
+		if (col > 0 && fTerminal.getChar(line, col) == '\000') {
+			col--; // the mark follows a wide character, and that is the cell it lives in
+		}
+		if (col < 0) {
+			return false;
+		}
+		char base = fTerminal.getChar(line, col);
+		if (base == 0 || base == ' ') {
+			return false;
+		}
+		String composed = Normalizer.normalize(base + new String(Character.toChars(mark)), Normalizer.Form.NFC);
+		if (composed.length() != 1) {
+			return false;
+		}
+		fTerminal.setChar(line, col, composed.charAt(0), fTerminal.getStyle(line, col));
+		return true;
+	}
+
+	/**
+	 * A wide character owns two cells. Overwriting either one leaves the other
+	 * stranded: a filler with nothing in front of it, or a glyph that now spills
+	 * over whatever was written next to it. Blanking the partner before the write
+	 * goes in keeps the line honest, which is what a terminal is expected to do.
+	 */
+	private void breakWideChar(int line, int col) {
+		if (col < 0 || col >= fColumns) {
+			return;
+		}
+		char c = fTerminal.getChar(line, col);
+		if (c == '\000') {
+			if (col > 0 && CharWidth.of(fTerminal.getChar(line, col - 1)) == 2) {
+				blank(line, col - 1);
+			}
+		} else if (CharWidth.of(c) == 2 && col + 1 < fColumns && fTerminal.getChar(line, col + 1) == '\000') {
+			blank(line, col + 1);
+		}
+	}
+
+	private void blank(int line, int col) {
+		fTerminal.setChar(line, col, ' ', fTerminal.getStyle(line, col));
+	}
+
+	/**
+	 * Length of the run of characters starting at <code>offset</code> that each
+	 * occupy exactly one cell, so that they can be copied in one block. Capped at
+	 * <code>max</code> cells. Zero when the run does not start with such a
+	 * character, which sends the caller down the code point by code point path.
+	 */
+	private static int narrowRun(char[] chars, int offset, int max) {
+		int n = 0;
+		while (n < max && offset + n < chars.length && !Character.isSurrogate(chars[offset + n])
+				&& CharWidth.of(chars[offset + n]) == 1) {
+			n++;
+		}
+		return n;
+	}
+
 	private int doLineWrap() {
 		int line;
 		line = toAbsoluteLine(fCursorLine);

terminal/bundles/org.eclipse.terminal.control/src/org/eclipse/terminal/internal/model/CharWidth.java

@@ -0,0 +1,122 @@
+/*******************************************************************************
+ * Copyright (c) 2026 Contributors to the Eclipse Foundation.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.terminal.internal.model;
+
+/**
+ * Display width of a code point, following Unicode Standard Annex #11
+ * (East Asian Width). Used by the emulator to keep its column arithmetic in
+ * step with what a terminal application assumes.
+ * <p>
+ * East Asian Wide (W) and Fullwidth (F) count as two columns. Ambiguous (A) is
+ * treated as narrow, as UAX #11 recommends for a context with no East Asian
+ * legacy encoding. Combining marks and non-printing characters count as zero.
+ */
+public final class CharWidth {
+
+	private CharWidth() {
+	}
+
+	/**
+	 * Wide and Fullwidth ranges from EastAsianWidth-17.0.0.txt, as inclusive
+	 * [start, end] pairs in ascending order. Everything not listed here defaults
+	 * to Narrow, which is what the file's {@code @missing} line specifies.
+	 */
+	private static final int[] WIDE_RANGES = {
+			0x01100, 0x0115F, 0x0231A, 0x0231B, 0x02329, 0x0232A, 0x023E9, 0x023EC,
+			0x023F0, 0x023F0, 0x023F3, 0x023F3, 0x025FD, 0x025FE, 0x02614, 0x02615,
+			0x02630, 0x02637, 0x02648, 0x02653, 0x0267F, 0x0267F, 0x0268A, 0x0268F,
+			0x02693, 0x02693, 0x026A1, 0x026A1, 0x026AA, 0x026AB, 0x026BD, 0x026BE,
+			0x026C4, 0x026C5, 0x026CE, 0x026CE, 0x026D4, 0x026D4, 0x026EA, 0x026EA,
+			0x026F2, 0x026F3, 0x026F5, 0x026F5, 0x026FA, 0x026FA, 0x026FD, 0x026FD,
+			0x02705, 0x02705, 0x0270A, 0x0270B, 0x02728, 0x02728, 0x0274C, 0x0274C,
+			0x0274E, 0x0274E, 0x02753, 0x02755, 0x02757, 0x02757, 0x02795, 0x02797,
+			0x027B0, 0x027B0, 0x027BF, 0x027BF, 0x02B1B, 0x02B1C, 0x02B50, 0x02B50,
+			0x02B55, 0x02B55, 0x02E80, 0x02E99, 0x02E9B, 0x02EF3, 0x02F00, 0x02FD5,
+			0x02FF0, 0x0303E, 0x03041, 0x03096, 0x03099, 0x030FF, 0x03105, 0x0312F,
+			0x03131, 0x0318E, 0x03190, 0x031E5, 0x031EF, 0x0321E, 0x03220, 0x03247,
+			0x03250, 0x0A48C, 0x0A490, 0x0A4C6, 0x0A960, 0x0A97C, 0x0AC00, 0x0D7A3,
+			0x0F900, 0x0FAFF, 0x0FE10, 0x0FE19, 0x0FE30, 0x0FE52, 0x0FE54, 0x0FE66,
+			0x0FE68, 0x0FE6B, 0x0FF01, 0x0FF60, 0x0FFE0, 0x0FFE6, 0x16FE0, 0x16FE4,
+			0x16FF0, 0x16FF6, 0x17000, 0x18CD5, 0x18CFF, 0x18D1E, 0x18D80, 0x18DF2,
+			0x1AFF0, 0x1AFF3, 0x1AFF5, 0x1AFFB, 0x1AFFD, 0x1AFFE, 0x1B000, 0x1B122,
+			0x1B132, 0x1B132, 0x1B150, 0x1B152, 0x1B155, 0x1B155, 0x1B164, 0x1B167,
+			0x1B170, 0x1B2FB, 0x1D300, 0x1D356, 0x1D360, 0x1D376, 0x1F004, 0x1F004,
+			0x1F0CF, 0x1F0CF, 0x1F18E, 0x1F18E, 0x1F191, 0x1F19A, 0x1F200, 0x1F202,
+			0x1F210, 0x1F23B, 0x1F240, 0x1F248, 0x1F250, 0x1F251, 0x1F260, 0x1F265,
+			0x1F300, 0x1F320, 0x1F32D, 0x1F335, 0x1F337, 0x1F37C, 0x1F37E, 0x1F393,
+			0x1F3A0, 0x1F3CA, 0x1F3CF, 0x1F3D3, 0x1F3E0, 0x1F3F0, 0x1F3F4, 0x1F3F4,
+			0x1F3F8, 0x1F43E, 0x1F440, 0x1F440, 0x1F442, 0x1F4FC, 0x1F4FF, 0x1F53D,
+			0x1F54B, 0x1F54E, 0x1F550, 0x1F567, 0x1F57A, 0x1F57A, 0x1F595, 0x1F596,
+			0x1F5A4, 0x1F5A4, 0x1F5FB, 0x1F64F, 0x1F680, 0x1F6C5, 0x1F6CC, 0x1F6CC,
+			0x1F6D0, 0x1F6D2, 0x1F6D5, 0x1F6D8, 0x1F6DC, 0x1F6DF, 0x1F6EB, 0x1F6EC,
+			0x1F6F4, 0x1F6FC, 0x1F7E0, 0x1F7EB, 0x1F7F0, 0x1F7F0, 0x1F90C, 0x1F93A,
+			0x1F93C, 0x1F945, 0x1F947, 0x1F9FF, 0x1FA70, 0x1FA7C, 0x1FA80, 0x1FA8A,
+			0x1FA8E, 0x1FAC6, 0x1FAC8, 0x1FAC8, 0x1FACD, 0x1FADC, 0x1FADF, 0x1FAEA,
+			0x1FAEF, 0x1FAF8, 0x20000, 0x2FFFD, 0x30000, 0x3FFFD
+	};
+
+	/** @return 0 for combining and non-printing, 2 for East Asian W/F, else 1 */
+	public static int of(int codePoint) {
+		if (codePoint < 0x0080) {
+			return codePoint < 0x20 || codePoint == 0x7F ? 0 : 1;
+		}
+		return isZeroWidth(codePoint) ? 0 : isWide(codePoint) ? 2 : 1;
+	}
+
+	/** @return the total display width of {@code s} */
+	public static int ofString(String s) {
+		return s.codePoints().map(CharWidth::of).sum();
+	}
+
+	/**
+	 * A {@code '\000'} means one of two things in a line of cells: the filler that
+	 * a wide character puts in the cell it also covers, which carries no text of
+	 * its own, or a cell that was never written or has been erased, which reads as
+	 * a space.
+	 *
+	 * @return whether the cell at {@code index} is the filler of the character
+	 *         before it
+	 */
+	public static boolean isFiller(CharSequence text, int index) {
+		return text.charAt(index) == '\000' && index > 0 && of(Character.codePointBefore(text, index)) == 2;
+	}
+
+	private static boolean isZeroWidth(int codePoint) {
+		// Hangul conjoining jamo vowels and trailing consonants: EAW lists them as
+		// neutral, but they combine into the leading consonant before them.
+		if (codePoint >= 0x1160 && codePoint <= 0x11FF) {
+			return true;
+		}
+		switch (Character.getType(codePoint)) {
+		case Character.NON_SPACING_MARK:
+		case Character.ENCLOSING_MARK:
+		case Character.CONTROL:
+		case Character.FORMAT:
+			return true;
+		default:
+			return false;
+		}
+	}
+
+	private static boolean isWide(int codePoint) {
+		int lo = 0, hi = WIDE_RANGES.length / 2 - 1;
+		while (lo <= hi) {
+			int mid = (lo + hi) >>> 1, i = mid * 2;
+			if (codePoint < WIDE_RANGES[i]) {
+				hi = mid - 1;
+			} else if (codePoint > WIDE_RANGES[i + 1])
+				lo = mid + 1;
+			else {
+				return true;
+			}
+		}
+		return false;
+	}
+}