From b5a2db87f91601c38734aff5369307db46e3c013 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 30 Jun 2026 07:52:13 +0200 Subject: [PATCH] fix(lexer): interpret \xHH hex and \NNN octal escapes in interpolating strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Double-quoted and other interpolating string literals dropped the backslash and leaked the literal characters for `\xHH` (hex byte) and `\NNN` (octal byte) escapes: `"\x41"` lexed to `"x41"` and `"\101"` to `"101"` instead of `"A"`. The `\uHHHH` / `\u{...}` Unicode escapes were broken the same way (`"\u{41}"` leaked `"u{41}"`). Only the basic escapes (`\n`, `\t`, `\0`, ...) were handled. scanStringSegment is the escape interpreter shared by every interpolating form — `"..."`, `%Q{...}`/`%{...}`, and interpolating heredocs all funnel through it (the latter two re-lex their body as a double-quoted string). This adds the missing cases there, matching MRI 4.0.5 byte-for-byte: - `\xHH`: 1–2 hex digits, greedy (`\x41`->"A", `\xff`->0xFF, `\x4`->0x04); a `\x` with no hex digit degrades to a literal `x` (MRI raises a SyntaxError, for which this entry point has no channel). - `\NNN` octal: 1–3 octal digits, greedy, masked to one byte (`\101`->"A", `\12`->"\n", `\377`->0xFF, `\400`->0x00). - `\uHHHH` (exactly four hex digits) and `\u{cp cp ...}` (whitespace- separated codepoints), each emitted as UTF-8; out-of-range / surrogate values fall back to U+FFFD. Single-quoted strings, `%q{}`, and the regexp pass-through are untouched. Adds string_escape_test.go covering hex/octal/unicode plus edge cases (max digits, single digit, greedy boundary, non-digit boundary, no hex digit after `\x`/`\u`, octal overflow, surrogate/out-of-range codepoints, empty/unterminated braces) and the `%Q`/`%{}`/interpolation-tail forms. Module coverage stays at 100%. Co-Authored-By: Claude Opus 4.8 --- lexer/lexer.go | 100 ++++++++++++++++++- lexer/string_escape_test.go | 192 ++++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 lexer/string_escape_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 800ee47..959b5ab 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -8,6 +8,7 @@ package lexer import ( "strings" + "unicode/utf8" "github.com/go-ruby-parser/parser/token" ) @@ -1284,6 +1285,75 @@ func isHexDigit(c byte) bool { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') } +// isOctalDigit reports whether c is an octal digit (0–7). +func isOctalDigit(c byte) bool { return c >= '0' && c <= '7' } + +// hexVal returns the numeric value of a single hex digit. The caller must have +// already established that c is a hex digit (isHexDigit). +func hexVal(c byte) int { + switch { + case c >= '0' && c <= '9': + return int(c - '0') + case c >= 'a' && c <= 'f': + return int(c-'a') + 10 + default: // 'A'..'F' + return int(c-'A') + 10 + } +} + +// appendUnicodeEscape resolves a `\u` escape at the cursor (the leading `\u` +// already consumed) and appends the UTF-8 encoding of each codepoint to b. Two +// forms are accepted, mirroring MRI: `\uHHHH` (exactly four hex digits) and +// `\u{H… H… …}` (one or more whitespace-separated codepoints). Malformed input +// (MRI raises a SyntaxError) is handled gracefully — only the well-formed +// prefix contributes bytes — since this lexer entry point has no error channel. +func (l *Lexer) appendUnicodeEscape(b []byte) []byte { + if l.peek() == '{' { + l.advance() // '{' + for { + for l.peek() == ' ' || l.peek() == '\t' { + l.advance() + } + if !isHexDigit(l.peek()) { + break + } + cp := 0 + for isHexDigit(l.peek()) { + cp = cp*16 + hexVal(l.advance()) + } + b = appendRune(b, cp) + } + for l.peek() != '}' && l.peek() != 0 { + l.advance() + } + if l.peek() == '}' { + l.advance() + } + return b + } + // `\uHHHH`: exactly four hex digits. With none present (MRI: SyntaxError) + // nothing is emitted, so a stray `\u` does not inject a NUL byte. + if !isHexDigit(l.peek()) { + return b + } + cp := 0 + for i := 0; i < 4 && isHexDigit(l.peek()); i++ { + cp = cp*16 + hexVal(l.advance()) + } + return appendRune(b, cp) +} + +// appendRune appends the UTF-8 encoding of codepoint cp to b. An out-of-range +// or surrogate value yields the U+FFFD replacement bytes, as Go's utf8 package +// does for invalid runes. +func appendRune(b []byte, cp int) []byte { + r := rune(cp) + if cp > 0x10FFFF || (cp >= 0xD800 && cp <= 0xDFFF) { + r = '�' + } + return utf8.AppendRune(b, r) +} + // percentDelimClose returns the closing delimiter for a %-literal opener: the // mate of a bracket pair, or the same character for a symmetric delimiter. func percentDelimClose(open byte) byte { @@ -1993,8 +2063,34 @@ func (l *Lexer) scanStringSegment() (string, bool) { b = append(b, '"') case 'e': b = append(b, 0x1b) - case '0': - b = append(b, 0) + case 'x': + // `\xHH`: 1–2 hex digits, greedy. MRI requires at least one hex + // digit (`\x`/`\xZ` is a SyntaxError); lacking an error channel + // here we fall back to emitting a literal 'x' so the byte stream + // stays well-defined. + if !isHexDigit(l.peek()) { + b = append(b, 'x') + continue + } + v := hexVal(l.advance()) + if isHexDigit(l.peek()) { + v = v*16 + hexVal(l.advance()) + } + b = append(b, byte(v)) + case '0', '1', '2', '3', '4', '5', '6', '7': + // `\NNN` octal: 1–3 octal digits, greedy; the leading digit was + // consumed as `esc`. The value is masked to a single byte + // (MRI: `\400` → byte 0x00). + v := int(esc - '0') + for i := 0; i < 2 && isOctalDigit(l.peek()); i++ { + v = v*8 + int(l.advance()-'0') + } + b = append(b, byte(v)) + case 'u': + // `\uHHHH` (exactly four hex digits) or `\u{H… H… …}` (one or + // more whitespace-separated codepoints). Each codepoint is + // emitted as UTF-8, matching MRI's `.bytes`. + b = l.appendUnicodeEscape(b) default: b = append(b, esc) } diff --git a/lexer/string_escape_test.go b/lexer/string_escape_test.go new file mode 100644 index 0000000..bf27d63 --- /dev/null +++ b/lexer/string_escape_test.go @@ -0,0 +1,192 @@ +package lexer + +import ( + "testing" + + "github.com/go-ruby-parser/parser/token" +) + +// firstStringLit tokenizes src and returns the literal of the first STRING, +// STRBEG, STRMID, or STREND token (the interpreted body of a double-quoted-style +// string), or ("", false) if none is found. +func firstStringLit(src string) (string, bool) { + for _, t := range New(src).Tokenize() { + switch t.Type { + case token.STRING, token.STRBEG, token.STRMID, token.STREND: + return t.Lit, true + } + } + return "", false +} + +// stringLits returns the literals of every STRING/STRBEG/STRMID/STREND token in +// src, in order — used to inspect a string split across interpolations. +func stringLits(src string) []string { + var out []string + for _, t := range New(src).Tokenize() { + switch t.Type { + case token.STRING, token.STRBEG, token.STRMID, token.STREND: + out = append(out, t.Lit) + } + } + return out +} + +// TestStringHexEscape: `\xHH` consumes one or two hex digits (greedy) and yields +// the corresponding byte, matching MRI 4.0.5. +func TestStringHexEscape(t *testing.T) { + cases := map[string]string{ + `"\x41"`: "A", // two digits + `"\x4"`: "\x04", // a single digit is enough + `"\xff"`: "\xff", // full byte, upper half + `"\xFF"`: "\xff", // uppercase hex digits + `"\xfff"`: "\xff" + "f", // greedy stops after two digits + `"\x41\x42"`: "AB", // adjacent escapes + `"a\x41b"`: "aAb", // surrounded by literal text + } + for src, want := range cases { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestStringHexEscapeNoDigit: `\x` with no following hex digit has no valid +// interpretation (MRI raises a SyntaxError). Lacking an error channel here the +// lexer degrades gracefully to a literal `x`, keeping the byte stream defined. +func TestStringHexEscapeNoDigit(t *testing.T) { + for src, want := range map[string]string{ + `"\xZ"`: "xZ", // non-hex follower + `"\x"`: "x", // end of string + } { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestStringOctalEscape: `\NNN` consumes one to three octal digits (greedy), +// masking the value to a single byte, matching MRI 4.0.5. +func TestStringOctalEscape(t *testing.T) { + cases := map[string]string{ + `"\101"`: "A", // three digits -> 0o101 = 65 + `"\12"`: "\n", // two digits -> 0o12 = 10 (newline) + `"\1"`: "\x01", // a single digit + `"\0"`: "\x00", // NUL (the previously-working single case) + `"\377"`: "\xff", // 0o377 = 255 + `"\400"`: "\x00", // overflow: 0o400 & 0xFF = 0 + `"\1010"`: "A0", // greedy stops after three digits + `"\08"`: "\x00" + "8", // non-octal digit ends the run + `"\779"`: "?9", // 0o77 = 63 ('?'), then literal 9 + `"a\101"`: "aA", // after literal text + } + for src, want := range cases { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestStringUnicodeEscape: `\uHHHH` (exactly four hex digits) and `\u{...}` +// (whitespace-separated codepoints) both expand to UTF-8 bytes, matching MRI. +func TestStringUnicodeEscape(t *testing.T) { + // bs is a single literal backslash; building the sources this way keeps the + // `\u` sequences intact (a bare `A` in Go source would be the compiler's + // own escape, not the two bytes the lexer must see). + bs := "\\" + cases := map[string]string{ + `"` + bs + `u0041"`: "A", // four-digit form -> 'A' + `"` + bs + `u00e9"`: "é", // four-digit, two-byte UTF-8 + `"a` + bs + `u0041b"`: "aAb", // four-digit among literal text + `"` + bs + `u{41}"`: "A", // brace form, single codepoint + `"` + bs + `u{41 42 43}"`: "ABC", // multiple codepoints + `"` + bs + `u{1F600}"`: "\U0001F600", // codepoint beyond the BMP + `"` + bs + `u{}"`: "", // empty braces -> nothing + `"` + bs + `u{ 41 }"`: "A", // surrounding whitespace tolerated + } + for src, want := range cases { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestStringUnicodeEscapeInvalid: out-of-range / surrogate codepoints fall back +// to U+FFFD, and a `\u` with no hex digit contributes nothing. +func TestStringUnicodeEscapeInvalid(t *testing.T) { + cases := map[string]string{ + `"\u{110000}"`: "�", // beyond U+10FFFF + `"\u{D800}"`: "�", // a surrogate half + `"\uZ"`: "Z", // no hex digit after \u: the Z stays as literal text + `"\u{ZZ}"`: "", // no hex digit inside braces + `"\u{41ZZ}"`: "A", // trailing junk before the close brace is skipped + `"\u{41`: "A", // unterminated brace form runs into end of input + } + for src, want := range cases { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestStringBasicEscapesStillWork guards the pre-existing escapes the new code +// sits beside, so the regression covers the whole switch. +func TestStringBasicEscapesStillWork(t *testing.T) { + cases := map[string]string{ + `"\n"`: "\n", + `"\t"`: "\t", + `"\r"`: "\r", + `"\a"`: "\a", + `"\b"`: "\b", + `"\v"`: "\v", + `"\f"`: "\f", + `"\e"`: "\x1b", + `"\s"`: " ", + `"\\"`: "\\", + `"\""`: "\"", + `"\q"`: "q", // unknown escape drops the backslash + } + for src, want := range cases { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +} + +// TestNumericEscapesInPercentQAndInterp: the new escapes apply to every +// interpolating string form (%Q, %{}, and the segment after an interpolation), +// since they all funnel through the same scanner. +func TestNumericEscapesInPercentQAndInterp(t *testing.T) { + if got, _ := firstStringLit(`%Q{\x41\101}`); got != "AA" { + t.Errorf(`%%Q{\x41\101}: got %q, want "AA"`, got) + } + if got, _ := firstStringLit(`%{A}`); got != "A" { + t.Errorf(`%%{A}: got %q, want "A"`, got) + } + // The trailing segment after an interpolation must interpret escapes too. + lits := stringLits("\"a#{x}\\x42\"") + if len(lits) < 2 || lits[len(lits)-1] != "B" { + t.Errorf("interpolation tail: lits=%q, want last segment %q", lits, "B") + } +} + +// TestNumericEscapesNotInSingleQuote: single-quoted strings keep `\x`/`\NNN` +// verbatim (only `\\` and `\'` are special), unchanged by this fix. +func TestNumericEscapesNotInSingleQuote(t *testing.T) { + for src, want := range map[string]string{ + `'\x41'`: `\x41`, + `'\101'`: `\101`, + `'A'`: `A`, + } { + got, ok := firstStringLit(src) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", src, got, ok, want) + } + } +}