From e769c4773c22f0fb43d097271a63ce1b49de0edc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 00:56:04 +0000 Subject: [PATCH 1/2] Add MarkupText.StandIn: text that carries its own ASCII stand-in (2.23.0) A StandInMarkup data layer names the ASCII text a reader whose client cannot show the covered text is sent instead. AsciiFold replaces the whole covered text with it when any character in it would be replaced, before the game's translations and the built-in stand-ins, and leaves it alone when the reader shows all of it. Stored on the wire as "standin". Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01BwFSYuSdkcBDL82H3jvQvG --- CHANGELOG.md | 10 ++++ MarkupString.Tests/AsciiFoldTests.cs | 62 +++++++++++++++++++++++ MarkupString/AsciiFold.cs | 70 +++++++++++++++++++++++++- MarkupString/Elements/ElementCodecs.cs | 3 ++ MarkupString/Elements/StandInMarkup.cs | 20 ++++++++ MarkupString/MarkupText.Elements.cs | 14 ++++++ MarkupString/PublicAPI.Unshipped.txt | 13 +++++ docs/layout.md | 5 ++ 8 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 MarkupString/Elements/StandInMarkup.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 521abb6..5617194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ and `MarkupString.Pueblo`. The packages share one version and are released toget The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 2.23.0 โ€” Unreleased + +### Added + +- **`MarkupText.StandIn(content, text)`** (`StandInMarkup`): the ASCII text a reader whose client cannot + show `content` is sent instead, so a ๐Ÿ“Œ can reach an ASCII client as `P` rather than `??`. + `AsciiFold` replaces the whole covered text when any character in it would be replaced, before a + game's translations and the built-in stand-ins, and leaves it alone for a reader who shows all of + it. No format writes anything for it, and it is stored on the wire as `standin`. + ## 2.22.1 โ€” Unreleased ### Fixed diff --git a/MarkupString.Tests/AsciiFoldTests.cs b/MarkupString.Tests/AsciiFoldTests.cs index 84f11f7..a0a3da7 100644 --- a/MarkupString.Tests/AsciiFoldTests.cs +++ b/MarkupString.Tests/AsciiFoldTests.cs @@ -117,4 +117,66 @@ public async Task Relayout_KeepsTheTreeAsGiven() var layout = relaid.Runs.SelectMany(run => run.Markups).OfType().Single(); await Assert.That(((Rule)layout.Root).Title!.ToPlainText()).IsEqualTo("a ยท b"); } + [Test] + public async Task StandIn_ReplacesTheWholeTextForAReaderWhoCannotShowIt() + { + var text = MarkupText.Concat([MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P"), MarkupText.Plain(" Rules")]); + + await Assert.That(AsciiFold.Default.Fold(text).Text).IsEqualTo("P Rules"); + await Assert.That(AsciiFold.Default.Fold(MarkupText.StandIn(MarkupText.Plain("โœ“ done"), "[x] done")).Text).IsEqualTo("[x] done"); + await Assert.That(AsciiFold.Default.Fold(MarkupText.StandIn(MarkupText.Plain("โœจ"), "")).Text).IsEqualTo(""); + } + + [Test] + public async Task StandIn_ComesBeforeTheGamesStandIns() + => await Assert.That(new AsciiFold([new("๐Ÿ“Œ", "*")]).Fold(MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P")).Text).IsEqualTo("P"); + + [Test] + public async Task StandIn_LeavesTextTheReaderShows() + { + var text = MarkupText.Concat([MarkupText.StandIn(MarkupText.Plain("cafรฉ"), "cafe!"), MarkupText.Plain(" ยท")]); + + await Assert.That(new AsciiFold(latin1: true).Fold(text).Text).IsEqualTo("cafรฉ ยท"); + await Assert.That(AsciiFold.Default.Fold(text).Text).IsEqualTo("cafe! *"); + } + + [Test] + public async Task StandIn_IsOneAcrossAColourChangeAndCarriesTheFirstCharactersMarkup() + { + var text = MarkupText.Concat([MarkupText.Plain("a"), MarkupText.StandIn(MarkupText.Concat([MarkupText.Wrap(Red, "โ†’"), MarkupText.Plain("โ€ฆ")]), "->"), MarkupText.Plain("b")]); + var folded = AsciiFold.Default.Fold(text); + + await Assert.That(folded.Text).IsEqualTo("a->b"); + await Assert.That(folded.Runs.Any(run => run.Start == 1 && run.Length == 2 && run.Markups.Contains(Red))).IsTrue(); + } + + [Test] + public async Task StandIn_SurvivesTheWire() + { + var text = MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P"); + var read = MarkupTextSerializer.Deserialize(MarkupTextSerializer.Serialize(text)); + + await Assert.That(AsciiFold.Default.Fold(read).Text).IsEqualTo("P"); + await Assert.That(read.Runs.Single().Markups.Contains(new StandInMarkup("P"))).IsTrue(); + } + + [Test] + public async Task StandIn_LaysOutAtItsOwnWidth() + { + var table = new Table([new TableColumn(MarkupText.Plain("")), new TableColumn(MarkupText.Plain("Title"))], + [ + [MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P"), MarkupText.Plain("Rules")], + [MarkupText.Plain("U"), MarkupText.Plain("News")], + ]); + var context = new LayoutContext { AsciiOnly = true, Fold = AsciiFold.Default }; + + var lines = BlockLayout.Lines(table, 30, context).Select(line => line.ToPlainText()).ToArray(); + + await Assert.That(lines.Single(line => line.Contains("Rules")).IndexOf("Rules")).IsEqualTo(lines.Single(line => line.Contains("News")).IndexOf("News")); + await Assert.That(lines.Any(line => line.TrimStart().StartsWith("P"))).IsTrue(); + } + + [Test] + public async Task StandIn_RefusesTextThatIsNotPrintableAscii() + => await Assert.That(() => MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "รฉ")).Throws(); } diff --git a/MarkupString/AsciiFold.cs b/MarkupString/AsciiFold.cs index 9117628..af3ed2c 100644 --- a/MarkupString/AsciiFold.cs +++ b/MarkupString/AsciiFold.cs @@ -21,7 +21,8 @@ namespace MarkupString; /// /// /// Markup is kept: a replacement carries the markup of the character it replaces. A point -/// () keeps its carrier. +/// () keeps its carrier. Text under a is replaced +/// whole by its stand-in when any of it would be replaced. /// /// public sealed class AsciiFold @@ -82,14 +83,33 @@ public MarkupText Fold(MarkupText text) if (IsAscii(source)) return text; var points = PointRanges(text.Runs); + var standIns = StandInRanges(text.Runs); var span = source.AsSpan(); var builder = new StringBuilder(source.Length); // Where each index of the old text lands in the new one, so the runs can follow. var map = new int[source.Length + 1]; var changed = false; + // The end of the stand-in being skipped, once its text is written. + var skipTo = 0; foreach (var range in Graphemes.Enumerate(span)) { var (start, end) = (range.Start.Value, range.End.Value); + if (start < skipTo) + { + // After the stand-in's text, so its first character's run is the one that holds it. + for (var i = start; i < end; i++) map[i] = builder.Length; + continue; + } + + if (StandInAt(standIns, start) is var (standInEnd, standIn) && NeedsFold(span[start..standInEnd], points, start)) + { + for (var i = start; i < end; i++) map[i] = builder.Length; + builder.Append(standIn); + skipTo = standInEnd; + changed = true; + continue; + } + var cluster = span[start..end]; var replacement = InPoint(points, start) ? null : Translate(cluster); for (var i = start; i < end; i++) map[i] = builder.Length; @@ -233,6 +253,54 @@ public MarkupText Fold(MarkupText text) return ranges; } + /// + /// The stretches of under a , in order, with touching runs + /// under the same one joined: a colour change inside a stand-in does not split it. + /// + private static List<(int Start, int End, string Text)>? StandInRanges(ImmutableArray runs) + { + List<(int Start, int End, string Text)>? ranges = null; + StandInMarkup? last = null; + foreach (var run in runs) + { + StandInMarkup? found = null; + foreach (var markup in run.Markups) + if (markup is StandInMarkup standIn) + { + found = standIn; + break; + } + if (found is null) + { + last = null; + continue; + } + + ranges ??= []; + if (found.Equals(last) && ranges[^1].End == run.Start) ranges[^1] = ranges[^1] with { End = run.End }; + else ranges.Add((run.Start, run.End, found.Text)); + last = found; + } + return ranges; + } + + /// The end and text of the stand-in that starts at , or null. + private static (int End, string Text)? StandInAt(List<(int Start, int End, string Text)>? standIns, int index) + { + if (standIns is null) return null; + foreach (var (start, end, text) in standIns) + if (start == index) return (end, text); + return null; + } + + /// Whether this fold would replace any character of , which starts at . + private bool NeedsFold(ReadOnlySpan text, List<(int Start, int End)>? points, int offset) + { + foreach (var range in Graphemes.Enumerate(text)) + if (!InPoint(points, offset + range.Start.Value) && Translate(text[range]) is not null) return true; + return false; + } + private static bool InPoint(List<(int Start, int End)>? points, int index) { if (points is null) return false; diff --git a/MarkupString/Elements/ElementCodecs.cs b/MarkupString/Elements/ElementCodecs.cs index 4d291ff..76a62ae 100644 --- a/MarkupString/Elements/ElementCodecs.cs +++ b/MarkupString/Elements/ElementCodecs.cs @@ -113,6 +113,9 @@ internal static class ElementCodecs if (m.Caption is not null) w.WriteString("ca", m.Caption); }, static e => new StatusMarkup(String(e, "n") ?? string.Empty, String(e, "m"), String(e, "ca"))), + new Codec("standin", + static (w, m) => w.WriteString("t", m.Text), + static e => new StandInMarkup(String(e, "t") ?? string.Empty)), ]; private static readonly Dictionary ByType = All.ToDictionary(c => c.MarkupType); diff --git a/MarkupString/Elements/StandInMarkup.cs b/MarkupString/Elements/StandInMarkup.cs new file mode 100644 index 0000000..39ba37d --- /dev/null +++ b/MarkupString/Elements/StandInMarkup.cs @@ -0,0 +1,20 @@ +namespace MarkupString; + +/// +/// The text a reader whose client cannot show the text it covers is sent instead: P for a ๐Ÿ“Œ, say, +/// where the built-in stand-ins would send ??. builds one. +/// +/// +/// +/// replaces the whole covered text with when any character in it +/// would be replaced, and leaves it alone when the reader shows all of it, so a Latin-1 reader keeps an +/// รฉ covered by one. The replacement carries the markup of the first character it replaces. It comes +/// before a game's own stand-ins and the built-in ones. +/// +/// +/// No format writes anything for it (): a reader that shows the text gets the +/// text. Two next to each other with the same text are one, as equal layers always are. +/// +/// +/// Printable ASCII, or empty to leave the covered text out. +public sealed record StandInMarkup(string Text) : IDataMarkup; diff --git a/MarkupString/MarkupText.Elements.cs b/MarkupString/MarkupText.Elements.cs index 62d00b7..45859ab 100644 --- a/MarkupString/MarkupText.Elements.cs +++ b/MarkupString/MarkupText.Elements.cs @@ -138,6 +138,20 @@ public static MarkupText Gauge(MarkupText content, string variable, string maxim return Wrap(new GaugeMarkup(variable, maximum, caption, color), content); } + /// + /// , sent as to a reader whose client cannot show it. + /// See . + /// + /// is not printable ASCII. + public static MarkupText StandIn(MarkupText content, string text) + { + ArgumentNullException.ThrowIfNull(content); + ArgumentNullException.ThrowIfNull(text); + foreach (var c in text) + if (c is < ' ' or > '~') throw new ArgumentException("A stand-in is printable ASCII.", nameof(text)); + return Wrap(new StandInMarkup(text), content); + } + /// /// in the client's status bar, standing in /// for a client with none. See . diff --git a/MarkupString/PublicAPI.Unshipped.txt b/MarkupString/PublicAPI.Unshipped.txt index 6054a51..9237e48 100644 --- a/MarkupString/PublicAPI.Unshipped.txt +++ b/MarkupString/PublicAPI.Unshipped.txt @@ -10,3 +10,16 @@ MarkupString.GradientTarget.Borders = 1 -> MarkupString.GradientTarget MarkupString.GradientTarget.Text = 2 -> MarkupString.GradientTarget MarkupString.Layout.Shaded.Target.get -> MarkupString.GradientTarget MarkupString.Layout.Shaded.Target.init -> void +MarkupString.StandInMarkup +MarkupString.StandInMarkup.$() -> MarkupString.StandInMarkup! +MarkupString.StandInMarkup.Deconstruct(out string! Text) -> void +MarkupString.StandInMarkup.Equals(MarkupString.StandInMarkup? other) -> bool +MarkupString.StandInMarkup.StandInMarkup(string! Text) -> void +MarkupString.StandInMarkup.Text.get -> string! +MarkupString.StandInMarkup.Text.init -> void +override MarkupString.StandInMarkup.Equals(object? obj) -> bool +override MarkupString.StandInMarkup.GetHashCode() -> int +override MarkupString.StandInMarkup.ToString() -> string! +static MarkupString.StandInMarkup.operator !=(MarkupString.StandInMarkup? left, MarkupString.StandInMarkup? right) -> bool +static MarkupString.StandInMarkup.operator ==(MarkupString.StandInMarkup? left, MarkupString.StandInMarkup? right) -> bool +static MarkupString.MarkupText.StandIn(MarkupString.MarkupText! content, string! text) -> MarkupString.MarkupText! diff --git a/docs/layout.md b/docs/layout.md index ba9eb81..f51b770 100644 --- a/docs/layout.md +++ b/docs/layout.md @@ -515,6 +515,11 @@ first, such as `ยท` to `" - "` or `โ€ฆ` to `"..."`; each key is one character ou printable ASCII. `fold.Fold(text)` folds a string or a `MarkupText`, whose markup stays on the characters it was on. +A text can carry its own stand-in: `MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P")` is sent as `P` to a +reader who cannot show the pin, where the built-in fold would send `??`. The whole covered text is +replaced when any character in it would be, and kept when the reader shows all of it. It comes before +the fold's own translations, and no format writes anything for it (`StandInMarkup` is an `IDataMarkup`). + Given as `LayoutContext.Fold`, the fold reaches the text inside every block (`Block.MapText`) before the block is measured, so a stand-in of another width still lines up. The tree a block carries is not changed. Text outside blocks is the caller's to fold. From 3e28367be0c625ee198cde32098ecebd1da41053 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 01:16:17 +0000 Subject: [PATCH 2/2] Keep points inside a replaced stand-in; ignore a stand-in whose text is not printable ASCII Points a stand-in covers keep their carriers and markup. A StandInMarkup built directly or read from the wire with non-ASCII text no longer reaches an ASCII reader. Stand-in lookup walks the ordered ranges once. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01BwFSYuSdkcBDL82H3jvQvG --- MarkupString.Tests/AsciiFoldTests.cs | 24 +++++++++++++++++ MarkupString/AsciiFold.cs | 39 ++++++++++++++++++---------- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/MarkupString.Tests/AsciiFoldTests.cs b/MarkupString.Tests/AsciiFoldTests.cs index a0a3da7..d8113b6 100644 --- a/MarkupString.Tests/AsciiFoldTests.cs +++ b/MarkupString.Tests/AsciiFoldTests.cs @@ -176,6 +176,30 @@ public async Task StandIn_LaysOutAtItsOwnWidth() await Assert.That(lines.Any(line => line.TrimStart().StartsWith("P"))).IsTrue(); } + [Test] + public async Task StandIn_KeepsThePointsItCovers() + { + var after = AsciiFold.Default.Fold(MarkupText.StandIn(MarkupText.Concat([MarkupText.Plain("๐Ÿ“Œ"), MarkupText.Sound("ping.wav")]), "P")); + var before = AsciiFold.Default.Fold(MarkupText.StandIn(MarkupText.Concat([MarkupText.Sound("ping.wav"), MarkupText.Plain("๐Ÿ“Œ")]), "P")); + + await Assert.That(after.Text).IsEqualTo("P" + MarkupText.PointCarrier); + await Assert.That(after.Runs.Any(run => run.Start == 1 && run.Length == 1 && run.Markups.Any(m => m is SoundMarkup))).IsTrue(); + await Assert.That(before.Text).IsEqualTo(MarkupText.PointCarrier + "P"); + await Assert.That(before.Runs.Any(run => run.Start == 0 && run.Length == 1 && run.Markups.Any(m => m is SoundMarkup))).IsTrue(); + } + + [Test] + public async Task StandIn_IsIgnoredWhenItsTextIsNotPrintableAscii() + => await Assert.That(AsciiFold.Default.Fold(MarkupText.Wrap(new StandInMarkup("รฉ"), MarkupText.Plain("๐Ÿ“Œ"))).Text).IsEqualTo("??"); + + [Test] + public async Task StandIn_ReplacesEachOfMany() + { + var text = MarkupText.Concat(Enumerable.Range(0, 1000).SelectMany(_ => new[] { MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "P"), MarkupText.Plain(" ") })); + + await Assert.That(AsciiFold.Default.Fold(text).Text).IsEqualTo(string.Concat(Enumerable.Repeat("P ", 1000))); + } + [Test] public async Task StandIn_RefusesTextThatIsNotPrintableAscii() => await Assert.That(() => MarkupText.StandIn(MarkupText.Plain("๐Ÿ“Œ"), "รฉ")).Throws(); diff --git a/MarkupString/AsciiFold.cs b/MarkupString/AsciiFold.cs index af3ed2c..234a346 100644 --- a/MarkupString/AsciiFold.cs +++ b/MarkupString/AsciiFold.cs @@ -89,24 +89,33 @@ public MarkupText Fold(MarkupText text) // Where each index of the old text lands in the new one, so the runs can follow. var map = new int[source.Length + 1]; var changed = false; - // The end of the stand-in being skipped, once its text is written. + // The end of the stand-in being replaced, and its text until it is written. var skipTo = 0; + string? pending = null; + // The first stand-in that may still start here; they are in order, as the graphemes are. + var nextStandIn = 0; foreach (var range in Graphemes.Enumerate(span)) { var (start, end) = (range.Start.Value, range.End.Value); - if (start < skipTo) + if (start >= skipTo && StandInAt(standIns, ref nextStandIn, start) is var (standInEnd, standIn) + && NeedsFold(span[start..standInEnd], points, start)) { - // After the stand-in's text, so its first character's run is the one that holds it. - for (var i = start; i < end; i++) map[i] = builder.Length; - continue; + skipTo = standInEnd; + pending = standIn; + changed = true; } - if (StandInAt(standIns, start) is var (standInEnd, standIn) && NeedsFold(span[start..standInEnd], points, start)) + if (start < skipTo) { + // A point keeps its carrier. The first character that is not one is written as the stand-in's + // text, so its run holds the text, and the rest are left out. for (var i = start; i < end; i++) map[i] = builder.Length; - builder.Append(standIn); - skipTo = standInEnd; - changed = true; + if (InPoint(points, start)) builder.Append(span[start..end]); + else if (pending is not null) + { + builder.Append(pending); + pending = null; + } continue; } @@ -264,8 +273,10 @@ public MarkupText Fold(MarkupText text) foreach (var run in runs) { StandInMarkup? found = null; + // One whose text is not printable ASCII (built directly, or read from the wire) is ignored, so the + // covered text folds as any other would. foreach (var markup in run.Markups) - if (markup is StandInMarkup standIn) + if (markup is StandInMarkup standIn && IsPrintableAscii(standIn.Text)) { found = standIn; break; @@ -285,12 +296,12 @@ public MarkupText Fold(MarkupText text) } /// The end and text of the stand-in that starts at , or null. - private static (int End, string Text)? StandInAt(List<(int Start, int End, string Text)>? standIns, int index) + /// moves past every stand-in that starts before . + private static (int End, string Text)? StandInAt(List<(int Start, int End, string Text)>? standIns, ref int next, int index) { if (standIns is null) return null; - foreach (var (start, end, text) in standIns) - if (start == index) return (end, text); - return null; + while (next < standIns.Count && standIns[next].Start < index) next++; + return next < standIns.Count && standIns[next].Start == index ? (standIns[next].End, standIns[next].Text) : null; } /// Whether this fold would replace any character of , which starts at .