Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion lexer/lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ func (l *Lexer) lexToken() token.Token {
case c == ':' && l.peek2() == '"':
// Quoted symbol, possibly interpolated: :"foo bar", :"a#{x}".
return l.lexSymbol(spaceBefore, line, col)
case c == ':' && l.peek2() == '\'':
// Single-quoted symbol: :'foo.bar', :'data-remote' (no interpolation).
return l.lexSymbol(spaceBefore, line, col)
}

// Operators and delimiters.
Expand Down Expand Up @@ -616,6 +619,15 @@ func (l *Lexer) lexSymbol(spaceBefore bool, line, col int) token.Token {
if l.peek() == '"' {
return l.lexQuotedSymbol(spaceBefore, line, col)
}
// Single-quoted symbols: :'foo.bar', :'data-remote' — no interpolation, the
// body's only escapes are \' and \\, exactly as a single-quoted string.
if l.peek() == '\'' {
str := l.lexSingleQuote(spaceBefore, line, col)
if str.Type == token.ILLEGAL {
return str
}
return token.Token{Type: token.SYMBOL, Lit: str.Lit, Line: line, Col: col, SpaceBefore: spaceBefore}
}
// Operator-method symbols: :+, :<=>, :[]=, …
if op := symbolOpAt(l.src, l.pos); op != "" {
for range op {
Expand Down Expand Up @@ -720,7 +732,9 @@ func (l *Lexer) lexGvar(spaceBefore bool, line, col int) token.Token {
l.advance() // '$'
start := l.pos
switch c := l.peek(); {
case c == '~' || c == '&' || c == '`' || c == '\'':
case isSpecialGvar(c):
// Single-character special globals: $~ $& $` $' $! $@ $/ $\ $; $, $.
// $< $> $? $* $$ $: $" $0 $+ (and the like). Each is exactly one byte.
l.advance()
case c >= '1' && c <= '9':
for l.peek() >= '0' && l.peek() <= '9' {
Expand Down Expand Up @@ -1479,6 +1493,18 @@ func isDigit(c byte) bool { return c >= '0' && c <= '9' }
func isIdentStart(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') }
func isIdentPart(c byte) bool { return isIdentStart(c) || isDigit(c) }

// isSpecialGvar reports whether c is one of Ruby's single-character special
// global-variable names that follow `$` (e.g. $! $@ $/ $\ $; $, $. $< $> $?
// $* $$ $: $" $+ $~ $& $` $'). The digit case ($1.., $0) is handled separately.
func isSpecialGvar(c byte) bool {
switch c {
case '~', '&', '`', '\'', '!', '@', '/', '\\', ';', ',', '.',
'<', '>', '?', '*', '$', ':', '"', '+':
return true
}
return false
}

func stripUnderscores(s string) string {
if !hasUnderscore(s) {
return s
Expand Down
118 changes: 118 additions & 0 deletions lexer/lexer_round1_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package lexer

import (
"testing"

"github.com/go-ruby-parser/parser/token"
)

// firstToken tokenizes src and returns its first non-EOF token.
func firstToken(t *testing.T, src string) token.Token {
t.Helper()
toks := New(src).Tokenize()
if len(toks) == 0 {
t.Fatalf("Tokenize(%q): no tokens", src)
}
return toks[0]
}

// TestSpecialGlobalVars covers the single-character special global variables that
// previously lexed as ILLEGAL. Each is one GVAR token whose Lit keeps the `$`.
func TestSpecialGlobalVars(t *testing.T) {
specials := []string{
"$$", "$!", "$@", "$/", "$\\", "$;", "$,", "$.", "$<", "$>",
"$?", "$*", "$\"", "$:", "$+", "$~", "$&", "$`", "$'",
}
for _, s := range specials {
tok := firstToken(t, s)
if tok.Type != token.GVAR {
t.Errorf("%q: type=%s, want GVAR", s, tok.Type)
}
if tok.Lit != s {
t.Errorf("%q: Lit=%q, want %q", s, tok.Lit, s)
}
}
}

// TestSpecialGlobalIsExactlyOneChar: a special global is one byte, so a token
// follows immediately ($$x is $$ then x).
func TestSpecialGlobalBoundary(t *testing.T) {
toks := New("$$x").Tokenize()
if toks[0].Type != token.GVAR || toks[0].Lit != "$$" {
t.Fatalf("$$x: first token=%v", toks[0])
}
if toks[1].Type != token.IDENT || toks[1].Lit != "x" {
t.Errorf("$$x: second token=%v, want IDENT x", toks[1])
}
}

// TestNumberedAndNamedGlobalsUnchanged: $1.., $0, and named globals still lex.
func TestNumberedAndNamedGlobals(t *testing.T) {
for _, c := range []struct{ src, lit string }{
{"$1", "$1"}, {"$42", "$42"}, {"$0", "$0"}, {"$stdout", "$stdout"},
} {
tok := firstToken(t, c.src)
if tok.Type != token.GVAR || tok.Lit != c.lit {
t.Errorf("%q: got %v, want GVAR %q", c.src, tok, c.lit)
}
}
}

// TestBareDollarIllegal: a lone `$` with no following name is still ILLEGAL.
func TestBareDollarIllegal(t *testing.T) {
tok := firstToken(t, "$ ")
if tok.Type != token.ILLEGAL {
t.Errorf("`$ `: type=%s, want ILLEGAL", tok.Type)
}
}

// TestSpecialGlobalInInterpolation: $$ inside #{} lexes as GVAR (regression: it
// used to break the whole string).
func TestSpecialGlobalInInterpolation(t *testing.T) {
toks := New(`"PID-#{$$}"`).Tokenize()
var sawGvar bool
for _, tk := range toks {
if tk.Type == token.GVAR && tk.Lit == "$$" {
sawGvar = true
}
if tk.Type == token.ILLEGAL {
t.Fatalf(`"PID-#{$$}": produced ILLEGAL token %v`, tk)
}
}
if !sawGvar {
t.Errorf(`"PID-#{$$}": no $$ GVAR token in %v`, toks)
}
}

// TestSingleQuotedSymbol covers :'…' symbols (no interpolation), including the
// \' and \\ escapes a single-quoted string honors.
func TestSingleQuotedSymbol(t *testing.T) {
for _, c := range []struct{ src, lit string }{
{`:'foo.bar'`, "foo.bar"},
{`:'data-remote'`, "data-remote"},
{`:'simple'`, "simple"},
{`:'a\'b'`, "a'b"},
{`:'a\\b'`, `a\b`},
{`:'with space'`, "with space"},
} {
tok := firstToken(t, c.src)
if tok.Type != token.SYMBOL || tok.Lit != c.lit {
t.Errorf("%q: got %v, want SYMBOL %q", c.src, tok, c.lit)
}
}
}

// TestUnterminatedSingleQuotedSymbol: an unterminated :'… is ILLEGAL.
func TestUnterminatedSingleQuotedSymbol(t *testing.T) {
tok := firstToken(t, ":'unterminated")
if tok.Type != token.ILLEGAL {
t.Errorf(":'unterminated: type=%s, want ILLEGAL", tok.Type)
}
}

// TestDoubleQuotedSymbolUnchanged: :"…" still works (plain and interpolated).
func TestDoubleQuotedSymbolUnchanged(t *testing.T) {
if tok := firstToken(t, `:"plain"`); tok.Type != token.SYMBOL || tok.Lit != "plain" {
t.Errorf(`:"plain": got %v`, tok)
}
}
Loading
Loading