Skip to content

Share LaTeX token and mode boundaries #336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Happypig375 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Share LaTeX token and mode boundaries #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions CSharpMath.Core.Tests/Atom/LaTeXParserTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1665,5 +1665,87 @@ public void TestErrorSurrogates() {
Assert.Equal(expected.Replace("\r", null), actual);
}
}
[Theory]
[InlineData("plain", 0, (int)LaTeXTokenKind.RawText, 0, 5, "plain")]
[InlineData("x \r\n", 1, (int)LaTeXTokenKind.Whitespace, 1, 3, " \r\n")]
[InlineData("\\", 0, (int)LaTeXTokenKind.ControlSymbol, 0, 1, "\\")]
[InlineData(@"\alpha+", 0, (int)LaTeXTokenKind.ControlWord, 0, 6, @"\alpha")]
[InlineData(@"\@name!", 0, (int)LaTeXTokenKind.ControlSymbol, 0, 2, @"\@")]
[InlineData(@"\%", 0, (int)LaTeXTokenKind.ControlSymbol, 0, 2, @"\%")]
[InlineData("{", 0, (int)LaTeXTokenKind.GroupOpen, 0, 1, "{")]
[InlineData("}", 0, (int)LaTeXTokenKind.GroupClose, 0, 1, "}")]
[InlineData("$", 0, (int)LaTeXTokenKind.InlineMathDelimiter, 0, 1, "$")]
[InlineData("$$", 0, (int)LaTeXTokenKind.DisplayMathDelimiter, 0, 2, "$$")]
[InlineData("$$$$", 0, (int)LaTeXTokenKind.InvalidDollarRun, 0, 4, "$$$$")]
[InlineData("\U0001F600\\alpha", 2, (int)LaTeXTokenKind.ControlWord, 2, 6, @"\alpha")]
public void SharedLexerReadAtPreservesUtf16Spans(string source, int offset,
int kind, int start, int length, string text) {
var token = LaTeXTokenizer.ReadAt(source, offset);

Assert.Equal((LaTeXTokenKind)kind, token.Kind);
Assert.Equal(start, token.Start);
Assert.Equal(length, token.Length);
Assert.Equal(start + length, token.End);
Assert.Equal(text, token.Text);
Assert.Same(source, token.Source);
}

[Theory]
[InlineData("\\", 1)]
[InlineData(@"\@name+", 2)]
[InlineData(@"\alpha@beta+", 6)]
[InlineData(@"\alpha**", 6)]
[InlineData(@"\alpha==", 6)]
[InlineData(@"\alpha''", 6)]
[InlineData(@"\@*'", 2)]
[InlineData(@"\*alpha", 2)]
public void SharedLexerUsesTeXControlSequenceBoundaries(string source, int expectedLength) =>
Assert.Equal(expectedLength, LaTeXTokenizer.ReadCommandLength(source.AsSpan()));

[Fact]
public void SharedLexerTokenizeIteratesEveryUtf16CodeUnit() {
var source = "\U0001F600raw \t\\@cmd*\\${}$$ $$$\\";
var expected = new[] {
(LaTeXTokenKind.RawText, 0, 5, "\U0001F600raw"),
(LaTeXTokenKind.Whitespace, 5, 2, " \t"),
(LaTeXTokenKind.ControlSymbol, 7, 2, @"\@"),
(LaTeXTokenKind.RawText, 9, 4, "cmd*"),
(LaTeXTokenKind.ControlSymbol, 13, 2, @"\$"),
(LaTeXTokenKind.GroupOpen, 15, 1, "{"),
(LaTeXTokenKind.GroupClose, 16, 1, "}"),
(LaTeXTokenKind.DisplayMathDelimiter, 17, 2, "$$"),
(LaTeXTokenKind.Whitespace, 19, 1, " "),
(LaTeXTokenKind.InvalidDollarRun, 20, 3, "$$$"),
(LaTeXTokenKind.ControlSymbol, 23, 1, "\\"),
};

var tokens = LaTeXTokenizer.Tokenize(source);

Assert.Equal(expected.Length, tokens.Count);
var nextStart = 0;
for (var i = 0; i < tokens.Count; i++) {
var token = tokens[i];
Assert.Equal(expected[i].Item1, token.Kind);
Assert.Equal(expected[i].Item2, token.Start);
Assert.Equal(expected[i].Item3, token.Length);
Assert.Equal(expected[i].Item4, token.Text);
Assert.Equal(nextStart, token.Start);
nextStart = token.End;
}
Assert.Equal(source.Length, nextStart);
Assert.Empty(LaTeXTokenizer.Tokenize(string.Empty));
}

[Fact]
public void MathParserPreservesEscapedDollarBracesAndNestedGroups() {
var list = ParseLaTeX(@"x{{\$\{\alpha\}}}");

Assert.Collection(list,
CheckAtom<Variable>("x"),
CheckAtom<Ordinary>("$"),
CheckAtom<Open>("{"),
CheckAtom<Variable>("α"),
CheckAtom<Close>("}"));
}
}
}
29 changes: 28 additions & 1 deletion CSharpMath.Rendering.Text.Tests/TextLaTeXParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,5 +503,32 @@ public void Error(string badInput, string expected) {
Assert.Null(atom);
Assert.Equal(expected.Replace("\r", null), actual);
}

[Theory]
[InlineData(@"\@name", @"\@", 3)]
[InlineData(@"\notacommand@beta", @"\notacommand", 13)]
[InlineData(@"\notacommand*", @"\notacommand", 13)]
[InlineData(@"\notacommand=", @"\notacommand", 13)]
[InlineData(@"\notacommand'", @"\notacommand", 13)]
public void TextParserReportsSharedControlSequenceBoundariesAtNonzeroOffsets(
string inputCommand, string expectedCommand, int position) {
var source = "x" + inputCommand + "+";

var (atom, error) = TextLaTeXParser.TextAtomFromLaTeX(source);

Assert.Null(atom);
Assert.Equal($"Error: Invalid command {expectedCommand}\n{source}\n{new string(' ', position - 1)}\u2191 (pos {position})", error);
}

[Theory]
[InlineData(@"x\alpha@beta+", @"x\alpha @beta+")]
[InlineData(@"x\alpha*+", @"x\alpha *+")]
[InlineData(@"x\alpha=+", @"x\alpha =+")]
[InlineData(@"x\alpha'+", @"x\alpha '+")]
public void TextParserLeavesControlWordSuffixesForText(string source, string expected) {
var atom = Parse(source);

Assert.Equal(expected, TextLaTeXParser.TextAtomToLaTeX(atom).ToString());
}
}
}
}
134 changes: 55 additions & 79 deletions CSharpMath.Rendering/Text/TextLaTeXParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public static Result<TextAtom> TextAtomFromLaTeX(string latexSource) {
if (string.IsNullOrEmpty(latexSource))
return new TextAtom.List(Array.Empty<TextAtom>());
int endAt = 0;
bool? displayMath = null;
var mode = LaTeXMode.Text;
var mathLaTeX = new StringBuilder();
bool backslashEscape = false;
bool afterCommand = false; // ignore spaces after command
Expand All @@ -60,42 +60,28 @@ public static Result<TextAtom> TextAtomFromLaTeX(string latexSource) {
breaker.AddBreakingEngine(engine);
breaker.BreakWords(latexSource);

Result TransitionMathMode(LaTeXModeBoundary boundary, int mathEndAt, ref int errorEndAt, TextAtomListBuilder atoms) {
var currentMode = mode;
if (LaTeXModeTransition.TryTransition(currentMode, boundary, out var nextMode) is string error)
return error;
if (currentMode != LaTeXMode.Text && nextMode == LaTeXMode.Text) {
if (atoms.Math(mathLaTeX.ToString(), currentMode == LaTeXMode.DisplayMath, mathEndAt, ref errorEndAt).Error is string mathError)
return mathError;
mathLaTeX.Clear();
}
mode = nextMode;
return Ok();
}
Result CheckDollarCount(int startAt, ref int endAt, TextAtomListBuilder atoms) {
switch (dollarCount) {
case 0:
break;
case 1:
dollarCount = 0;
switch (displayMath) {
case true:
return "Cannot close display math mode with $";
case false:
if (atoms.Math(mathLaTeX.ToString(), false, startAt, ref endAt).Error is string error)
return error;
mathLaTeX.Clear();
displayMath = null;
break;
case null:
displayMath = false;
break;
}
break;
return TransitionMathMode(LaTeXModeBoundary.InlineDollar, startAt, ref endAt, atoms);
case 2:
dollarCount = 0;
switch (displayMath) {
case true:
if (atoms.Math(mathLaTeX.ToString(), true, startAt - 1, ref endAt).Error is string error)
return error;
mathLaTeX.Clear();
displayMath = null;
break;
case false:
return "Cannot close inline math mode with $$";
case null:
displayMath = true;
break;
}
break;
return TransitionMathMode(LaTeXModeBoundary.DisplayDollar, startAt - 1, ref endAt, atoms);
default:
return "Invalid number of $: " + dollarCount;
}
Expand Down Expand Up @@ -126,6 +112,13 @@ Result<TextAtom> ReadArgumentAtom(ReadOnlySpan<char> latexInput) {
return BuildBreakList(latexInput, argAtoms, ++i, true, '\0')
.Bind(index => { i = index; return argAtoms.Build(); });
}
var sharedCommandEnd = endAt;
var sharedCommandName = textSection.ToString();
if (backslashEscape && startAt > 0 && latexSource[startAt - 1] == '\\') {
var sharedCommand = LaTeXTokenizer.ReadAt(latexSource, startAt - 1);
sharedCommandName = latexSource.Substring(startAt, sharedCommand.Length - 1);
sharedCommandEnd = sharedCommand.End;
}
SpanResult<char> ReadArgumentString(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> section) {
afterCommand = false;
if (!NextSection(latexInput, ref section) || section.IsNot('{')) return Err("Missing {");
Expand Down Expand Up @@ -157,7 +150,7 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
atoms.TextLength = startAt;
if (textSection.Is('$')) {
if (backslashEscape)
if (displayMath != null) mathLaTeX.Append(@"\$");
if (mode != LaTeXMode.Text) mathLaTeX.Append(@"\$");
else atoms.Text("$");
else {
dollarCount++;
Expand All @@ -166,8 +159,9 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
backslashEscape = false;
} else {
{ if (CheckDollarCount(startAt, ref endAt, atoms).Error is string error) return error; }
switch (backslashEscape, displayMath) {
case (false, { }):
switch (backslashEscape, mode) {
case (false, LaTeXMode.InlineMath):
case (false, LaTeXMode.DisplayMath):
//Unescaped text section, inside display/inline math mode
switch (textSection) {
case var _ when textSection.Is('$'):
Expand All @@ -181,7 +175,7 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
}
afterCommand = false;
break;
case (false, null):
case (false, LaTeXMode.Text):
//Unescaped text section, not inside display/inline math mode
switch (textSection) {
case var _ when stopChar > 0 && textSection[0] == stopChar:
Expand Down Expand Up @@ -245,74 +239,57 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
}
afterCommand = false;
break;
case (true, { }):
case (true, LaTeXMode.InlineMath):
case (true, LaTeXMode.DisplayMath):
//Escaped text section but in inline/display math mode
switch (textSection) {
case var _ when textSection.Is('$'):
throw new InvalidCodePathException("The $ case should have been accounted for.");
case var _ when textSection.Is('('):
return displayMath switch {
true => "Cannot open inline math mode in display math mode",
false => "Cannot open inline math mode in inline math mode",
null => throw new InvalidCodePathException("displayMath is null. This switch should not be hit."),
};
if (TransitionMathMode(LaTeXModeBoundary.InlineCommandOpen, startAt, ref endAt, atoms).Error is string inlineOpenError)
return inlineOpenError;
break;
case var _ when textSection.Is(')'):
switch (displayMath) {
case true:
return "Cannot close inline math mode in display math mode";
case false:
if (atoms.Math(mathLaTeX.ToString(), false, startAt, ref endAt).Error is string mathError)
return mathError;
mathLaTeX.Clear();
displayMath = null;
break;
case null:
throw new InvalidCodePathException("displayMath is null. This switch should not be hit.");
}
if (TransitionMathMode(LaTeXModeBoundary.InlineCommandClose, startAt, ref endAt, atoms).Error is string inlineError)
return inlineError;
break;
case var _ when textSection.Is('['):
return displayMath switch {
true => "Cannot open display math mode in display math mode",
false => "Cannot open display math mode in inline math mode",
null => throw new InvalidCodePathException("displayMath is null. This switch should not be hit."),
};
if (TransitionMathMode(LaTeXModeBoundary.DisplayCommandOpen, startAt, ref endAt, atoms).Error is string displayOpenError)
return displayOpenError;
break;
case var _ when textSection.Is(']'):
switch (displayMath) {
case true:
if (atoms.Math(mathLaTeX.ToString(), true, startAt, ref endAt).Error is string mathError)
return mathError;
mathLaTeX.Clear();
displayMath = null;
break;
case false:
return "Cannot close display math mode in inline math mode";
default:
throw new InvalidCodePathException("displayMath is null. This switch should not be hit.");
}
if (TransitionMathMode(LaTeXModeBoundary.DisplayCommandClose, startAt, ref endAt, atoms).Error is string displayError)
return displayError;
break;
default:
mathLaTeX.Append('\\').Append(textSection);
break;
}
backslashEscape = false;
break;
case (true, null):
case (true, LaTeXMode.Text):
//Escaped text section and not in inline/display math mode
afterCommand = true;
switch (textSection.ToString()) {
switch (sharedCommandName) {
case var _ when wordKind == WordKind.Whitespace: //control space
atoms.ControlSpace();
break;
case "(":
displayMath = false;
if (TransitionMathMode(LaTeXModeBoundary.InlineCommandOpen, startAt, ref endAt, atoms).Error is string inlineOpenError)
return inlineOpenError;
break;
case ")":
return "Cannot close inline math mode outside of math mode";
if (TransitionMathMode(LaTeXModeBoundary.InlineCommandClose, startAt, ref endAt, atoms).Error is string inlineCloseError)
return inlineCloseError;
break;
case "[":
displayMath = true;
if (TransitionMathMode(LaTeXModeBoundary.DisplayCommandOpen, startAt, ref endAt, atoms).Error is string displayOpenError)
return displayOpenError;
break;
case "]":
return "Cannot close display math mode outside of math mode";
if (TransitionMathMode(LaTeXModeBoundary.DisplayCommandClose, startAt, ref endAt, atoms).Error is string displayCloseError)
return displayCloseError;
break;
case "\\":
atoms.Break();
break;
Expand Down Expand Up @@ -416,9 +393,8 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
atoms.Text(replaceResult);
break;
case var command:
if (displayMath != null) mathLaTeX.Append(command); //don't eat the command when parsing math
else return $@"Invalid command \{command}";
break;
endAt = sharedCommandEnd;
return $@"Invalid command \{command}";
}
backslashEscape = false;
break;
Expand All @@ -435,7 +411,7 @@ Result<Color> ReadColor(ReadOnlySpan<char> latexInput, ref ReadOnlySpan<char> se
if (error != null) return LaTeXParser.HelpfulErrorMessage(error, latexSource, endAt);
error = CheckDollarCount(latexSource.Length, ref endAt, globalAtoms).Error;
if (error != null) return LaTeXParser.HelpfulErrorMessage(error, latexSource, endAt);
if (displayMath != null) return LaTeXParser.HelpfulErrorMessage("Math mode was not terminated", latexSource, endAt);
if (mode != LaTeXMode.Text) return LaTeXParser.HelpfulErrorMessage("Math mode was not terminated", latexSource, endAt);
return globalAtoms.Build();
}
public static StringBuilder TextAtomToLaTeX(TextAtom atom, StringBuilder? b = null) {
Expand Down Expand Up @@ -504,4 +480,4 @@ public static StringBuilder TextAtomToLaTeX(TextAtom atom, StringBuilder? b = nu
}
}
}
}
}
3 changes: 3 additions & 0 deletions CSharpMath/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("CSharpMath.Rendering")]
[assembly: InternalsVisibleTo("CSharpMath.Core.Tests")]
10 changes: 1 addition & 9 deletions CSharpMath/Atom/Dictionary.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,7 @@ public IEnumerator<KeyValuePair<string, TValue>> GetEnumerator() =>
// https://tug.org/texinfohtml/latex2e.html#g_t_005cmakeatletter_0026-_005cmakeatother
static bool IsAsciiLetter(char c) => 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z';

static int SplitCommand(ReadOnlySpan<char> chars) {
System.Diagnostics.Debug.Assert(chars[0] == '\\');
var splitIndex = 1;
if (splitIndex < chars.Length)
if (IsAsciiLetter(chars[splitIndex])) {
do splitIndex++; while (splitIndex < chars.Length && IsAsciiLetter(chars[splitIndex]));
} else splitIndex++;
return splitIndex;
}
static int SplitCommand(ReadOnlySpan<char> chars) => LaTeXTokenizer.ReadCommandLength(chars);
/// <summary>Tries to find a command at the beginning of <see cref="char"/>s, returning the
/// <typeparamref name="TValue"/> corresponding to the command Key, and the length of the command.</summary>
public Result<(TValue Result, int SplitIndex)> TryLookup(ReadOnlySpan<char> chars) {
Expand Down
Loading
Loading