From 532a384144c1bc1ba3964d218e2ae9442bc266d0 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 11 Sep 2026 07:25:03 +0200 Subject: [PATCH 1/2] fix(lexer): a backslash escapes the next character in a %w/%W/%i/%I list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a word list a backslash escapes the character after it, so that character is an ordinary character of the current word: it never separates two words and never opens or closes the list. The lexer was escape-blind, which cost three distinct defects. - An escaped space split the word. `%w[a\ b c]` lexed as three words ("a\", "b", "c") where MRI has two ("a b", "c"). ruby/spec's Dir fixtures contain a directory literally named `special/test\ +()[]{}` written inside a %w list, so every Dir spec that enumerates it saw a phantom `+()[]{}` entry. - An escaped delimiter ended the list. `%w[a\]b]` was a parse error, and `%w[a\[b]` an "unterminated %-array literal". - An escaped backslash was doubled: `%w[a\\b]` yielded `a\\b`, not `a\b`. MRI's rule (parse.y, tokadd_string) follows from the backslash branch being tested before the word-splitting and term/paren branches. In a non-interpolating %w/%i the backslash is KEPT literally except before whitespace, a delimiter, or another backslash, where it escapes a literal character instead — so `%w[a\tb]` is the four characters a \ t b and not a tab. In an interpolating %W/%I the full double-quote escape set applies, with one exception: MRI's QWORDS test precedes its EXPAND one, so a backslash before a newline is a literal newline rather than a line continuation. Splitting moves from the parser (strings.Fields, which cannot see an escape) to the lexer, which still knows the delimiters; the WORDS/SYMBOLS token gains a Words field and keeps its raw body in Lit. %I words are now unescaped too, so `%I[a\ b]` is :"a b" rather than the four characters a \ space b. Every rule above was checked against the installed MRI (ruby 4.0.5) rather than read off parse.y alone: 62 word-list sources were run through both `ruby` and this parser and compared as raw bytes, all 62 identical. The three further cases in that set are interpolating and fold to a StrInterp node instead of a constant, and were checked by hand. --- lexer/lexer.go | 138 ++++++++++++++++++++++++++++++++++++++++++++++--- parser.go | 4 +- token/token.go | 7 +-- 3 files changed, 137 insertions(+), 12 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index dd7ac07..ef7c27c 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1426,9 +1426,15 @@ func (l *Lexer) atPercentArray() bool { } // lexPercentArray lexes a %w/%i/%W/%I array literal. The non-interpolating -// %w/%i forms become a single WORDS/SYMBOLS token whose Lit the parser splits. -// The interpolating %W/%I forms are spliced into the equivalent array of -// (interpolated) string/symbol elements. Bracket delimiters nest. +// %w/%i forms become a WORDS/SYMBOLS token carrying both the raw body (Lit) and +// the words it splits into (Words). The interpolating %W/%I forms are spliced +// into the equivalent array of (interpolated) string/symbol elements. Bracket +// delimiters nest. +// +// A backslash escapes the character after it, which is therefore neither a +// nesting bracket nor the terminator: `%w[a\]b]` is the single word `a]b`, not +// an unterminated list (MRI parse.y, tokadd_string — the `c == '\\'` branch +// consumes the next character before the term/paren tests can see it). func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { l.advance() // % kind := l.advance() @@ -1436,10 +1442,21 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { closing := percentDelimClose(open) depth := 1 var content []byte + unterminated := token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore} for { c := l.peek() if c == 0 { - return token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore} + return unterminated + } + if c == '\\' { + // Keep both bytes raw; the escape is resolved when the body is + // split into words, which is where the delimiters are still known. + content = append(content, l.advance()) + if l.peek() == 0 { + return unterminated + } + content = append(content, l.advance()) + continue } if open != closing && c == open { depth++ @@ -1460,7 +1477,79 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { if kind == 'i' { tt = token.SYMBOLS } - return token.Token{Type: tt, Lit: string(content), Line: line, Col: col, SpaceBefore: spaceBefore} + return token.Token{ + Type: tt, + Lit: string(content), + Words: splitWordList(string(content), open, closing), + Line: line, + Col: col, + SpaceBefore: spaceBefore, + } +} + +// isWordListSpace reports whether c separates two words of a %-list. It is +// MRI's ISSPACE: space, tab, newline, vertical tab, form feed, carriage return. +func isWordListSpace(c byte) bool { + switch c { + case ' ', '\t', '\n', '\v', '\f', '\r': + return true + } + return false +} + +// splitWordList splits the raw body of a NON-interpolating %w/%i list into its +// words, resolving escapes as MRI does (parse.y, tokadd_string with +// func = str_sword: STR_FUNC_QWORDS, no STR_FUNC_EXPAND, no STR_FUNC_ESCAPE). +// +// Two rules matter and both follow from the backslash branch being tested +// before the word-splitting one: +// +// - Only UNESCAPED whitespace separates words. `%w[a\ b c]` is the two words +// "a b" and "c" — the escaped space is an ordinary character of the first +// word. (ruby -e 'p %w[a\ b c]' => ["a b", "c"]) +// - The backslash itself survives EXCEPT before whitespace, a delimiter, or +// another backslash, where it escapes a literal character instead. So +// `%w[a\tb]` is the four characters a \ t b and NOT a tab, while +// `%w[a\\b]` is the three characters a \ b. +// (ruby -e 'p %w[a\tb]' => ["a\\tb"]; ruby -e 'p %w[a\\b]' => ["a\\b"]) +func splitWordList(body string, open, closing byte) []string { + words := []string{} + var cur []byte + flush := func() { + if len(cur) > 0 { + words = append(words, string(cur)) + cur = nil + } + } + for i := 0; i < len(body); i++ { + c := body[i] + if c == '\\' && i+1 < len(body) { + d := body[i+1] + switch { + case isWordListSpace(d): + // The backslash is dropped and the whitespace becomes a + // literal character of the current word, never a separator. + cur = append(cur, d) + case d == '\\': + cur = append(cur, '\\') + case d == closing || (open != closing && d == open): + cur = append(cur, d) + default: + // Everything else keeps the backslash verbatim; there is no + // escape set here, only the four cases above. + cur = append(cur, '\\', d) + } + i++ + continue + } + if isWordListSpace(c) { + flush() + continue + } + cur = append(cur, c) + } + flush() + return words } // isPercentDelim reports whether b can open a %-literal. MRI accepts any @@ -1670,7 +1759,7 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l toks = append(toks, token.Token{Type: token.COMMA, Lit: ",", Line: line, Col: col}) } if symbols && !strings.Contains(w, "#{") { - toks = append(toks, token.Token{Type: token.SYMBOL, Lit: w, Line: line, Col: col}) + toks = append(toks, token.Token{Type: token.SYMBOL, Lit: unescapeInterpWord(w), Line: line, Col: col}) continue } for _, t := range New(`"` + wrapHeredocDQ(w) + `"`).Tokenize() { @@ -1689,8 +1778,32 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l return toks[0] } +// unescapeInterpWord resolves the escapes of a %I word that carries no +// interpolation, so `%I[a\ b]` is the symbol :"a b" and not the four characters +// a \ space b. A %I list interpolates, so its escape set is the double-quoted +// one; the word is re-lexed as a double-quoted string to reuse that decoder. +// (ruby -e 'p %I[a\ b]' => [:"a b"]) +func unescapeInterpWord(w string) string { + if !strings.Contains(w, `\`) { + return w + } + toks := New(`"` + wrapHeredocDQ(w) + `"`).Tokenize() + if len(toks) == 2 && toks[0].Type == token.STRING && toks[1].Type == token.EOF { + return toks[0].Lit + } + return w +} + // splitPercentWords splits a %W/%I body on whitespace, keeping whitespace that -// falls inside a #{…} interpolation as part of the word. +// is escaped, or that falls inside a #{…} interpolation, as part of the word. +// +// Escapes are left in place rather than resolved: each word is re-lexed as a +// double-quoted string, whose decoder already implements the full escape set +// MRI applies to an interpolating list (parse.y, str_dword: read_escape). The +// one divergence is a backslash before a newline — MRI's QWORDS test precedes +// its EXPAND test, so in %W that is a literal newline rather than the line +// continuation it would be in a string — so it is rewritten to `\n` here. +// (ruby -e 'p %W[a\b]' => ["a\nb"]) func splitPercentWords(s string) []string { var words []string var cur []byte @@ -1703,7 +1816,16 @@ func splitPercentWords(s string) []string { } for i := 0; i < len(s); i++ { c := s[i] - if depth == 0 && (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v') { + if depth == 0 && c == '\\' && i+1 < len(s) { + if d := s[i+1]; d == '\n' { + cur = append(cur, '\\', 'n') + } else { + cur = append(cur, '\\', d) + } + i++ + continue + } + if depth == 0 && isWordListSpace(c) { flush() continue } diff --git a/parser.go b/parser.go index 99fb211..f3170f0 100644 --- a/parser.go +++ b/parser.go @@ -2917,7 +2917,9 @@ func (p *Parser) parsePrimary() ast.Node { case token.WORDS, token.SYMBOLS: p.advance() elems := []ast.Node{} - for _, w := range strings.Fields(t.Lit) { + // The lexer splits the body: separating whitespace can only be told + // from an escaped literal space there, where the delimiters are known. + for _, w := range t.Words { if t.Type == token.SYMBOLS { elems = append(elems, &ast.SymbolLit{Name: w}) } else { diff --git a/token/token.go b/token/token.go index f7408e6..2fa50f3 100644 --- a/token/token.go +++ b/token/token.go @@ -28,8 +28,8 @@ const ( LABEL // name: in a hash literal REGEXP // /pattern/flags (Lit = pattern source, Flags = matched flag letters) XSTRING // `cmd` / %x{cmd} backtick command literal (Lit = raw command source) - WORDS // %w[…] word-array literal (Lit = raw whitespace-separated content) - SYMBOLS // %i[…] symbol-array literal (Lit = raw whitespace-separated content) + WORDS // %w[…] word-array literal (Lit = raw body, Words = the split words) + SYMBOLS // %i[…] symbol-array literal (Lit = raw body, Words = the split words) // Keywords. DEF @@ -155,7 +155,8 @@ var Keywords = map[string]Type{ type Token struct { Type Type Lit string - Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens + Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens + Words []string // the words of a WORDS/SYMBOLS literal, already split and unescaped; only set for those two types Line int Col int SpaceBefore bool // whitespace immediately preceded this token (MRI spaceSeen) From 997e65aa6cbdbf21459bc280468696e8d2524f53 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 11 Sep 2026 07:27:58 +0200 Subject: [PATCH 2/2] test(lexer): pin the %w/%W/%i/%I escape rules against MRI Four tables covering the non-interpolating (%w, %i) and interpolating (%W, %I) forms: escaped space, escaped delimiter for all six delimiter pairs, escaped backslash, escaped tab and newline, the backslashes %w keeps verbatim because it has no escape set, the double-quote escape set %W applies instead, and the unescaped whitespace and bracket nesting that must keep behaving as before. Every expected value is the value the installed MRI (ruby 4.0.5) prints for the same source; each was run through `ruby -e 'p '` and compared as raw bytes, so a failure here is a divergence from Ruby rather than from taste. The MRI output is quoted beside each case. Also covers the scanner's new end-of-input branch: a list ending on a trailing backslash, or on an escaped terminator, is unterminated rather than accepted. --- lexer/lexer.go | 9 +-- percent_word_escape_test.go | 153 ++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 percent_word_escape_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index ef7c27c..3b3cec3 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1787,11 +1787,10 @@ func unescapeInterpWord(w string) string { if !strings.Contains(w, `\`) { return w } - toks := New(`"` + wrapHeredocDQ(w) + `"`).Tokenize() - if len(toks) == 2 && toks[0].Type == token.STRING && toks[1].Type == token.EOF { - return toks[0].Lit - } - return w + // The caller has already established that the word holds no interpolation, + // and the list scanner never leaves an unpaired backslash in it, so the + // re-lex always yields one string token followed by EOF. + return New(`"` + wrapHeredocDQ(w) + `"`).Tokenize()[0].Lit } // splitPercentWords splits a %W/%I body on whitespace, keeping whitespace that diff --git a/percent_word_escape_test.go b/percent_word_escape_test.go new file mode 100644 index 0000000..7cc5038 --- /dev/null +++ b/percent_word_escape_test.go @@ -0,0 +1,153 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" +) + +// The tables below pin the escape rules of the four word-list forms. Every +// `want` is the value the installed MRI prints for the same source — each case +// was run through `ruby -e 'p '` and compared byte for byte — so a +// disagreement here is a divergence from Ruby, not a matter of taste. +// +// The rules, from MRI's parse.y (tokadd_string): a backslash escapes the +// character after it, which therefore neither separates two words nor opens or +// closes the list. In a NON-interpolating %w/%i the backslash is otherwise KEPT +// literally — it escapes a character only before whitespace, a delimiter, or +// another backslash. In an interpolating %W/%I the full double-quote escape set +// applies instead, except that a backslash before a newline is a literal +// newline rather than a line continuation. + +// TestPercentWordListEscapes covers %w (non-interpolating word list). +func TestPercentWordListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + // An escaped space is a literal space inside the word, never a + // separator. This is the case ruby/spec's Dir fixtures depend on: a + // directory literally named `special/test +()[]{}`. + {`x = %w[a\ b c]`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[special/test\ +()[]{} other]`, []string{"special/test +()[]{}", "other"}}, // ["special/test +()[]{}", "other"] + {`x = %w|a\ b c|`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[ a\ b c ]`, []string{"a b", "c"}}, // ["a b", "c"] + {`x = %w[a\ b\ c]`, []string{"a b c"}}, // ["a b c"] + {`x = %w[a\ b]`, []string{"a ", "b"}}, // ["a ", "b"] + {`x = %w[\ ]`, []string{" "}}, // [" "] + {"x = %w[a\\\tb]", []string{"a\tb"}}, // ["a\tb"] (escaped TAB) + {"x = %w[a\\\nb]", []string{"a\nb"}}, // ["a\nb"] (escaped newline) + + // An escaped delimiter is a literal character: it neither closes the + // list nor opens a nested level. + {`x = %w[a\]b]`, []string{"a]b"}}, // ["a]b"] + {`x = %w[a\[b]`, []string{"a[b"}}, // ["a[b"] + {`x = %w(a\)b)`, []string{"a)b"}}, // ["a)b"] + {`x = %w(a\(b)`, []string{"a(b"}}, // ["a(b"] + {`x = %w{a\}b}`, []string{"a}b"}}, // ["a}b"] + {`x = %w{a\{b}`, []string{"a{b"}}, // ["a{b"] + {`x = %wb>`, []string{"a>b"}}, // ["a>b"] + {`x = %w!a\!b!`, []string{"a!b"}}, // ["a!b"] + + // An escaped backslash is ONE literal backslash. + {`x = %w[a\\b]`, []string{`a\b`}}, // ["a\\b"] + {`x = %w[\\]`, []string{`\`}}, // ["\\"] + {`x = %w[a\b\\c]`, []string{`a\b\c`}}, // ["a\\b\\c"] + + // Every other backslash is kept verbatim: %w has no escape set, so + // `\t` is the two characters `\` and `t`, NOT a tab. + {`x = %w[a\tb]`, []string{`a\tb`}}, // ["a\\tb"] + {`x = %w[a\nb]`, []string{`a\nb`}}, // ["a\\nb"] + {`x = %w[a\ub]`, []string{`a\ub`}}, // ["a\\ub"] + {`x = %w[a\#b]`, []string{`a\#b`}}, // ["a\\#b"] + {`x = %w[a\x41b]`, []string{`a\x41b`}}, // ["a\\x41b"] + + // Unescaped whitespace still separates, and brackets still nest. + {`x = %w[a b c]`, []string{"a", "b", "c"}}, // ["a", "b", "c"] + {"x = %w[a\tb]", []string{"a", "b"}}, // ["a", "b"] + {"x = %w[a\nb]", []string{"a", "b"}}, // ["a", "b"] + {`x = %w[a[b]c]`, []string{"a[b]c"}}, // ["a[b]c"] + {`x = %w[a[b]c d]`, []string{"a[b]c", "d"}}, // ["a[b]c", "d"] + {`x = %w(a(b)c)`, []string{"a(b)c"}}, // ["a(b)c"] + {`x = %w[]`, nil}, // [] + {`x = %w[ ]`, nil}, // [] + } { + wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentInterpWordListEscapes covers %W (interpolating word list), where +// the full double-quote escape set applies. +func TestPercentInterpWordListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + {`x = %W[a\ b]`, []string{"a b"}}, // ["a b"] + {`x = %W[\ ]`, []string{" "}}, // [" "] + {`x = %W[a\]b]`, []string{"a]b"}}, // ["a]b"] + {`x = %W{a\}b}`, []string{"a}b"}}, // ["a}b"] + {`x = %W[a\\b]`, []string{`a\b`}}, // ["a\\b"] + {`x = %W[a\tb]`, []string{"a\tb"}}, // ["a\tb"] + {`x = %W[a\nb]`, []string{"a\nb"}}, // ["a\nb"] + {`x = %W[a\sb]`, []string{"a b"}}, // ["a b"] + {`x = %W[a\eb]`, []string{"a\x1bb"}}, // ["a\eb"] + {`x = %W[a\#b]`, []string{"a#b"}}, // ["a#b"] + {`x = %W[a\x41b]`, []string{"aAb"}}, // ["aAb"] + {`x = %W[a\101b]`, []string{"aAb"}}, // ["aAb"] + // An escaped `#{` is literal text, not an interpolation. + {`x = %W[a\#{1}b]`, []string{"a#{1}b"}}, // ["a\#{1}b"] + // A backslash before a newline is a literal newline here, NOT the line + // continuation it would be inside a double-quoted string: MRI tests + // STR_FUNC_QWORDS before STR_FUNC_EXPAND. + {"x = %W[a\\\nb]", []string{"a\nb"}}, // ["a\nb"] + {"x = %W[a\\\tb]", []string{"a\tb"}}, // ["a\tb"] + {`x = %W[a b c]`, []string{"a", "b", "c"}}, + } { + wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentSymbolListEscapes covers %i and %I, which split exactly as %w and +// %W do and differ only in producing symbols. +func TestPercentSymbolListEscapes(t *testing.T) { + for _, tc := range []struct { + src string + want []string + }{ + // %i: non-interpolating, so the backslash survives except before + // whitespace, a delimiter, or another backslash. + {`x = %i[a\ b]`, []string{"a b"}}, // [:"a b"] + {`x = %i[\ ]`, []string{" "}}, // [:" "] + {`x = %i[a\]b c]`, []string{"a]b", "c"}}, // [:"a]b", :c] + {`x = %i[a\\b]`, []string{`a\b`}}, // [:"a\\b"] + {`x = %i[a\tb]`, []string{`a\tb`}}, // [:"a\\tb"] + {`x = %i[a b c]`, []string{"a", "b", "c"}}, + + // %I: interpolating, so the double-quote escape set applies. + {`x = %I[a\ b]`, []string{"a b"}}, // [:"a b"] + {`x = %I[x\ y\ z]`, []string{"x y z"}}, // [:"x y z"] + {`x = %I[a\]b]`, []string{"a]b"}}, // [:"a]b"] + {`x = %I[a\tb]`, []string{"a\tb"}}, // [:"a\tb"] + {`x = %I[a b c]`, []string{"a", "b", "c"}}, + } { + wantSymbols(t, tc.src, parseRHS(t, tc.src), tc.want) + } +} + +// TestPercentWordListUnterminated guards the scanner's end-of-input handling +// now that a backslash consumes the character after it: a list that ends on a +// trailing backslash is unterminated, not silently accepted. +func TestPercentWordListUnterminated(t *testing.T) { + for _, src := range []string{ + `x = %w[a\`, + `x = %W[a\`, + `x = %i[a\`, + `x = %I[a\`, + `x = %w[a\]`, // the `\]` is escaped, so the list never closes + } { + if _, err := parser.Parse(src); err == nil { + t.Errorf("expected a parse error for %q, got none", src) + } + } +}