Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 129 additions & 8 deletions lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -1426,20 +1426,37 @@ func (l *Lexer) atPercentArray() bool {
}

// lexPercentArray lexes a %w/%i/%W/%I array literal. The non-interpolating
// %w/%i forms become a single WORDS/SYMBOLS token whose Lit the parser splits.
// The interpolating %W/%I forms are spliced into the equivalent array of
// (interpolated) string/symbol elements. Bracket delimiters nest.
// %w/%i forms become a WORDS/SYMBOLS token carrying both the raw body (Lit) and
// the words it splits into (Words). The interpolating %W/%I forms are spliced
// into the equivalent array of (interpolated) string/symbol elements. Bracket
// delimiters nest.
//
// A backslash escapes the character after it, which is therefore neither a
// nesting bracket nor the terminator: `%w[a\]b]` is the single word `a]b`, not
// an unterminated list (MRI parse.y, tokadd_string — the `c == '\\'` branch
// consumes the next character before the term/paren tests can see it).
func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token {
l.advance() // %
kind := l.advance()
open := l.advance()
closing := percentDelimClose(open)
depth := 1
var content []byte
unterminated := token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore}
for {
c := l.peek()
if c == 0 {
return token.Token{Type: token.ILLEGAL, Lit: "unterminated %-array literal", Line: line, Col: col, SpaceBefore: spaceBefore}
return unterminated
}
if c == '\\' {
// Keep both bytes raw; the escape is resolved when the body is
// split into words, which is where the delimiters are still known.
content = append(content, l.advance())
if l.peek() == 0 {
return unterminated
}
content = append(content, l.advance())
continue
}
if open != closing && c == open {
depth++
Expand All @@ -1460,7 +1477,79 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token {
if kind == 'i' {
tt = token.SYMBOLS
}
return token.Token{Type: tt, Lit: string(content), Line: line, Col: col, SpaceBefore: spaceBefore}
return token.Token{
Type: tt,
Lit: string(content),
Words: splitWordList(string(content), open, closing),
Line: line,
Col: col,
SpaceBefore: spaceBefore,
}
}

// isWordListSpace reports whether c separates two words of a %-list. It is
// MRI's ISSPACE: space, tab, newline, vertical tab, form feed, carriage return.
func isWordListSpace(c byte) bool {
switch c {
case ' ', '\t', '\n', '\v', '\f', '\r':
return true
}
return false
}

// splitWordList splits the raw body of a NON-interpolating %w/%i list into its
// words, resolving escapes as MRI does (parse.y, tokadd_string with
// func = str_sword: STR_FUNC_QWORDS, no STR_FUNC_EXPAND, no STR_FUNC_ESCAPE).
//
// Two rules matter and both follow from the backslash branch being tested
// before the word-splitting one:
//
// - Only UNESCAPED whitespace separates words. `%w[a\ b c]` is the two words
// "a b" and "c" — the escaped space is an ordinary character of the first
// word. (ruby -e 'p %w[a\ b c]' => ["a b", "c"])
// - The backslash itself survives EXCEPT before whitespace, a delimiter, or
// another backslash, where it escapes a literal character instead. So
// `%w[a\tb]` is the four characters a \ t b and NOT a tab, while
// `%w[a\\b]` is the three characters a \ b.
// (ruby -e 'p %w[a\tb]' => ["a\\tb"]; ruby -e 'p %w[a\\b]' => ["a\\b"])
func splitWordList(body string, open, closing byte) []string {
words := []string{}
var cur []byte
flush := func() {
if len(cur) > 0 {
words = append(words, string(cur))
cur = nil
}
}
for i := 0; i < len(body); i++ {
c := body[i]
if c == '\\' && i+1 < len(body) {
d := body[i+1]
switch {
case isWordListSpace(d):
// The backslash is dropped and the whitespace becomes a
// literal character of the current word, never a separator.
cur = append(cur, d)
case d == '\\':
cur = append(cur, '\\')
case d == closing || (open != closing && d == open):
cur = append(cur, d)
default:
// Everything else keeps the backslash verbatim; there is no
// escape set here, only the four cases above.
cur = append(cur, '\\', d)
}
i++
continue
}
if isWordListSpace(c) {
flush()
continue
}
cur = append(cur, c)
}
flush()
return words
}

// isPercentDelim reports whether b can open a %-literal. MRI accepts any
Expand Down Expand Up @@ -1670,7 +1759,7 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l
toks = append(toks, token.Token{Type: token.COMMA, Lit: ",", Line: line, Col: col})
}
if symbols && !strings.Contains(w, "#{") {
toks = append(toks, token.Token{Type: token.SYMBOL, Lit: w, Line: line, Col: col})
toks = append(toks, token.Token{Type: token.SYMBOL, Lit: unescapeInterpWord(w), Line: line, Col: col})
continue
}
for _, t := range New(`"` + wrapHeredocDQ(w) + `"`).Tokenize() {
Expand All @@ -1689,8 +1778,31 @@ func (l *Lexer) splicePercentInterp(content string, symbols, spaceBefore bool, l
return toks[0]
}

// unescapeInterpWord resolves the escapes of a %I word that carries no
// interpolation, so `%I[a\ b]` is the symbol :"a b" and not the four characters
// a \ space b. A %I list interpolates, so its escape set is the double-quoted
// one; the word is re-lexed as a double-quoted string to reuse that decoder.
// (ruby -e 'p %I[a\ b]' => [:"a b"])
func unescapeInterpWord(w string) string {
if !strings.Contains(w, `\`) {
return w
}
// The caller has already established that the word holds no interpolation,
// and the list scanner never leaves an unpaired backslash in it, so the
// re-lex always yields one string token followed by EOF.
return New(`"` + wrapHeredocDQ(w) + `"`).Tokenize()[0].Lit
}

// splitPercentWords splits a %W/%I body on whitespace, keeping whitespace that
// falls inside a #{…} interpolation as part of the word.
// is escaped, or that falls inside a #{…} interpolation, as part of the word.
//
// Escapes are left in place rather than resolved: each word is re-lexed as a
// double-quoted string, whose decoder already implements the full escape set
// MRI applies to an interpolating list (parse.y, str_dword: read_escape). The
// one divergence is a backslash before a newline — MRI's QWORDS test precedes
// its EXPAND test, so in %W that is a literal newline rather than the line
// continuation it would be in a string — so it is rewritten to `\n` here.
// (ruby -e 'p %W[a\<newline>b]' => ["a\nb"])
func splitPercentWords(s string) []string {
var words []string
var cur []byte
Expand All @@ -1703,7 +1815,16 @@ func splitPercentWords(s string) []string {
}
for i := 0; i < len(s); i++ {
c := s[i]
if depth == 0 && (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v') {
if depth == 0 && c == '\\' && i+1 < len(s) {
if d := s[i+1]; d == '\n' {
cur = append(cur, '\\', 'n')
} else {
cur = append(cur, '\\', d)
}
i++
continue
}
if depth == 0 && isWordListSpace(c) {
flush()
continue
}
Expand Down
4 changes: 3 additions & 1 deletion parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -2917,7 +2917,9 @@ func (p *Parser) parsePrimary() ast.Node {
case token.WORDS, token.SYMBOLS:
p.advance()
elems := []ast.Node{}
for _, w := range strings.Fields(t.Lit) {
// The lexer splits the body: separating whitespace can only be told
// from an escaped literal space there, where the delimiters are known.
for _, w := range t.Words {
if t.Type == token.SYMBOLS {
elems = append(elems, &ast.SymbolLit{Name: w})
} else {
Expand Down
153 changes: 153 additions & 0 deletions percent_word_escape_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package parser_test

import (
"testing"

"github.com/go-ruby-parser/parser"
)

// The tables below pin the escape rules of the four word-list forms. Every
// `want` is the value the installed MRI prints for the same source — each case
// was run through `ruby -e 'p <src>'` and compared byte for byte — so a
// disagreement here is a divergence from Ruby, not a matter of taste.
//
// The rules, from MRI's parse.y (tokadd_string): a backslash escapes the
// character after it, which therefore neither separates two words nor opens or
// closes the list. In a NON-interpolating %w/%i the backslash is otherwise KEPT
// literally — it escapes a character only before whitespace, a delimiter, or
// another backslash. In an interpolating %W/%I the full double-quote escape set
// applies instead, except that a backslash before a newline is a literal
// newline rather than a line continuation.

// TestPercentWordListEscapes covers %w (non-interpolating word list).
func TestPercentWordListEscapes(t *testing.T) {
for _, tc := range []struct {
src string
want []string
}{
// An escaped space is a literal space inside the word, never a
// separator. This is the case ruby/spec's Dir fixtures depend on: a
// directory literally named `special/test +()[]{}`.
{`x = %w[a\ b c]`, []string{"a b", "c"}}, // ["a b", "c"]
{`x = %w[special/test\ +()[]{} other]`, []string{"special/test +()[]{}", "other"}}, // ["special/test +()[]{}", "other"]
{`x = %w|a\ b c|`, []string{"a b", "c"}}, // ["a b", "c"]
{`x = %w[ a\ b c ]`, []string{"a b", "c"}}, // ["a b", "c"]
{`x = %w[a\ b\ c]`, []string{"a b c"}}, // ["a b c"]
{`x = %w[a\ b]`, []string{"a ", "b"}}, // ["a ", "b"]
{`x = %w[\ ]`, []string{" "}}, // [" "]
{"x = %w[a\\\tb]", []string{"a\tb"}}, // ["a\tb"] (escaped TAB)
{"x = %w[a\\\nb]", []string{"a\nb"}}, // ["a\nb"] (escaped newline)

// An escaped delimiter is a literal character: it neither closes the
// list nor opens a nested level.
{`x = %w[a\]b]`, []string{"a]b"}}, // ["a]b"]
{`x = %w[a\[b]`, []string{"a[b"}}, // ["a[b"]
{`x = %w(a\)b)`, []string{"a)b"}}, // ["a)b"]
{`x = %w(a\(b)`, []string{"a(b"}}, // ["a(b"]
{`x = %w{a\}b}`, []string{"a}b"}}, // ["a}b"]
{`x = %w{a\{b}`, []string{"a{b"}}, // ["a{b"]
{`x = %w<a\>b>`, []string{"a>b"}}, // ["a>b"]
{`x = %w!a\!b!`, []string{"a!b"}}, // ["a!b"]

// An escaped backslash is ONE literal backslash.
{`x = %w[a\\b]`, []string{`a\b`}}, // ["a\\b"]
{`x = %w[\\]`, []string{`\`}}, // ["\\"]
{`x = %w[a\b\\c]`, []string{`a\b\c`}}, // ["a\\b\\c"]

// Every other backslash is kept verbatim: %w has no escape set, so
// `\t` is the two characters `\` and `t`, NOT a tab.
{`x = %w[a\tb]`, []string{`a\tb`}}, // ["a\\tb"]
{`x = %w[a\nb]`, []string{`a\nb`}}, // ["a\\nb"]
{`x = %w[a\ub]`, []string{`a\ub`}}, // ["a\\ub"]
{`x = %w[a\#b]`, []string{`a\#b`}}, // ["a\\#b"]
{`x = %w[a\x41b]`, []string{`a\x41b`}}, // ["a\\x41b"]

// Unescaped whitespace still separates, and brackets still nest.
{`x = %w[a b c]`, []string{"a", "b", "c"}}, // ["a", "b", "c"]
{"x = %w[a\tb]", []string{"a", "b"}}, // ["a", "b"]
{"x = %w[a\nb]", []string{"a", "b"}}, // ["a", "b"]
{`x = %w[a[b]c]`, []string{"a[b]c"}}, // ["a[b]c"]
{`x = %w[a[b]c d]`, []string{"a[b]c", "d"}}, // ["a[b]c", "d"]
{`x = %w(a(b)c)`, []string{"a(b)c"}}, // ["a(b)c"]
{`x = %w[]`, nil}, // []
{`x = %w[ ]`, nil}, // []
} {
wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want)
}
}

// TestPercentInterpWordListEscapes covers %W (interpolating word list), where
// the full double-quote escape set applies.
func TestPercentInterpWordListEscapes(t *testing.T) {
for _, tc := range []struct {
src string
want []string
}{
{`x = %W[a\ b]`, []string{"a b"}}, // ["a b"]
{`x = %W[\ ]`, []string{" "}}, // [" "]
{`x = %W[a\]b]`, []string{"a]b"}}, // ["a]b"]
{`x = %W{a\}b}`, []string{"a}b"}}, // ["a}b"]
{`x = %W[a\\b]`, []string{`a\b`}}, // ["a\\b"]
{`x = %W[a\tb]`, []string{"a\tb"}}, // ["a\tb"]
{`x = %W[a\nb]`, []string{"a\nb"}}, // ["a\nb"]
{`x = %W[a\sb]`, []string{"a b"}}, // ["a b"]
{`x = %W[a\eb]`, []string{"a\x1bb"}}, // ["a\eb"]
{`x = %W[a\#b]`, []string{"a#b"}}, // ["a#b"]
{`x = %W[a\x41b]`, []string{"aAb"}}, // ["aAb"]
{`x = %W[a\101b]`, []string{"aAb"}}, // ["aAb"]
// An escaped `#{` is literal text, not an interpolation.
{`x = %W[a\#{1}b]`, []string{"a#{1}b"}}, // ["a\#{1}b"]
// A backslash before a newline is a literal newline here, NOT the line
// continuation it would be inside a double-quoted string: MRI tests
// STR_FUNC_QWORDS before STR_FUNC_EXPAND.
{"x = %W[a\\\nb]", []string{"a\nb"}}, // ["a\nb"]
{"x = %W[a\\\tb]", []string{"a\tb"}}, // ["a\tb"]
{`x = %W[a b c]`, []string{"a", "b", "c"}},
} {
wantStrings(t, tc.src, parseRHS(t, tc.src), tc.want)
}
}

// TestPercentSymbolListEscapes covers %i and %I, which split exactly as %w and
// %W do and differ only in producing symbols.
func TestPercentSymbolListEscapes(t *testing.T) {
for _, tc := range []struct {
src string
want []string
}{
// %i: non-interpolating, so the backslash survives except before
// whitespace, a delimiter, or another backslash.
{`x = %i[a\ b]`, []string{"a b"}}, // [:"a b"]
{`x = %i[\ ]`, []string{" "}}, // [:" "]
{`x = %i[a\]b c]`, []string{"a]b", "c"}}, // [:"a]b", :c]
{`x = %i[a\\b]`, []string{`a\b`}}, // [:"a\\b"]
{`x = %i[a\tb]`, []string{`a\tb`}}, // [:"a\\tb"]
{`x = %i[a b c]`, []string{"a", "b", "c"}},

// %I: interpolating, so the double-quote escape set applies.
{`x = %I[a\ b]`, []string{"a b"}}, // [:"a b"]
{`x = %I[x\ y\ z]`, []string{"x y z"}}, // [:"x y z"]
{`x = %I[a\]b]`, []string{"a]b"}}, // [:"a]b"]
{`x = %I[a\tb]`, []string{"a\tb"}}, // [:"a\tb"]
{`x = %I[a b c]`, []string{"a", "b", "c"}},
} {
wantSymbols(t, tc.src, parseRHS(t, tc.src), tc.want)
}
}

// TestPercentWordListUnterminated guards the scanner's end-of-input handling
// now that a backslash consumes the character after it: a list that ends on a
// trailing backslash is unterminated, not silently accepted.
func TestPercentWordListUnterminated(t *testing.T) {
for _, src := range []string{
`x = %w[a\`,
`x = %W[a\`,
`x = %i[a\`,
`x = %I[a\`,
`x = %w[a\]`, // the `\]` is escaped, so the list never closes
} {
if _, err := parser.Parse(src); err == nil {
t.Errorf("expected a parse error for %q, got none", src)
}
}
}
7 changes: 4 additions & 3 deletions token/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ const (
LABEL // name: in a hash literal
REGEXP // /pattern/flags (Lit = pattern source, Flags = matched flag letters)
XSTRING // `cmd` / %x{cmd} backtick command literal (Lit = raw command source)
WORDS // %w[…] word-array literal (Lit = raw whitespace-separated content)
SYMBOLS // %i[…] symbol-array literal (Lit = raw whitespace-separated content)
WORDS // %w[…] word-array literal (Lit = raw body, Words = the split words)
SYMBOLS // %i[…] symbol-array literal (Lit = raw body, Words = the split words)

// Keywords.
DEF
Expand Down Expand Up @@ -155,7 +155,8 @@ var Keywords = map[string]Type{
type Token struct {
Type Type
Lit string
Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens
Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens
Words []string // the words of a WORDS/SYMBOLS literal, already split and unescaped; only set for those two types
Line int
Col int
SpaceBefore bool // whitespace immediately preceded this token (MRI spaceSeen)
Expand Down