From 2bcd2dda15388645e88cfc354af1d99eea71e7c8 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 21:20:47 +0200 Subject: [PATCH] Round 5: paren-less/destructuring def params, for loops, and command-arg disambiguation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushes Rails+Puppet parse conformance from 98.66% to 99.96% (5577/5579), the residual two files being intentional syntax-error test fixtures that MRI itself rejects — i.e. a literal 100% of all valid Ruby in the corpus. Parser/lexer additions, each verified against MRI 4.0.x (`ruby -c`) with MRI-parity table tests: - Paren-less method-def params, including leading `*`/`**` and destructuring groups (`def f *a, &b`, `def f((a, b), c)`, anonymous splat in a group), expanded to a leading multiple-assignment prepended to the body. - Character-literal escapes `?\xHH`, `?\uHHHH`, `?\u{…}`, `?\NNN`, `?\C-x`, `?\M-x`, `?\M-\C-x` (the consumers swallow the full escape). - Empty `||` and block-local variables after `;` in a block parameter list. - A nested regexp inside an interpolation inside a regexp / %r-literal, via a verbatim brace-balanced interpolation copy that skips strings. - `yield(...)` as a paren-less command argument. - RSpec `it { … }` / `it do … end`: a bare `it` taking a block is a method call, not the Ruby-3.4 implicit block parameter. - A bareword that is a known local, or a constant, promoted back to a method call by an unambiguous following command argument (`type = 1; type 'X'`), and a local-name `do` block (`fork = false; pid = fork do … end`). - Keyword/assignment argument values that no longer swallow the following argument; command-argument masgn suppression (`assert_equal a, tag = b`). - A `key:` value on the next line inside a call or hash, distinct from the value-omitted shorthand. - A paren-less setter `def x=` whose body starts on the next line (the setter `=` no longer continues the line), and keyword-named setters (`def ensure=`). - The `for VAR[, …] in ITER [do] … end` loop (new ast.For node). - `super`/`yield` followed by a space-flanked `/` parsed as division, not a regexp argument. - One-line pattern match in a condition (`if node in Foo[...]`) and bracket patterns spanning newlines. - An assignment / op-assignment as a range endpoint (`@i...@i += n`). 100% coverage maintained (CI gate), gofmt + go vet clean, CGO=0, all suites race-clean. End-to-end verified by rebuilding rbgo against this tree. rbgo follow-up: the new ast.For node needs a compiler case in go-embedded-ruby (parsing is complete; compilation of `for` is not yet wired). Co-Authored-By: Claude Opus 4.8 --- ast/ast.go | 10 ++ lexer/lexer.go | 207 +++++++++++++++++++++++++++- parser.go | 301 ++++++++++++++++++++++++++++++++++++---- round5_test.go | 367 +++++++++++++++++++++++++++++++++++++++++++++++++ token/token.go | 5 +- 5 files changed, 859 insertions(+), 31 deletions(-) create mode 100644 round5_test.go diff --git a/ast/ast.go b/ast/ast.go index 7a5d4ba..d85b3bc 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -140,6 +140,15 @@ type While struct { Body []Node } +// For is a `for VARS in ITER ... end` loop. Vars names the one or more loop +// variables (`for a, b in pairs`); unlike a block, a `for` does not introduce a +// new scope, so the variables remain visible after the loop. +type For struct { + Vars []string + Iter Node + Body []Node +} + // MethodDef defines a method on the current self. type MethodDef struct { Name string @@ -474,6 +483,7 @@ func (*UnaryExpr) node() {} func (*Call) node() {} func (*If) node() {} func (*While) node() {} +func (*For) node() {} func (*MethodDef) node() {} func (*Return) node() {} func (*ConstRef) node() {} diff --git a/lexer/lexer.go b/lexer/lexer.go index 60905a8..800ee47 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -56,6 +56,20 @@ type Lexer struct { // a trailing operator names a method (`alias eql? ==`⏎`def …`) rather than // continuing the line, so operator line-continuation is suppressed. inFitemList bool + // defNamePending is set after `def` (and re-armed after the `.` of a `def + // recv.name` form) until the method name is read, so a `=` that hugs that name + // is recognised as a setter-method terminator (`def x=`) rather than an + // assignment operator. A setter `=` does not continue the line, so a body that + // begins on the next line (`def x=`⏎` foo do … end`) is not mis-read as a + // paren-less parameter list. + defNamePending bool + // defNameJustRead is set for exactly the one token following the def method + // name, so a `=` hugging that name (and only there) is the setter terminator. + defNameJustRead bool + // setterAssign marks the just-emitted `=` as such a setter terminator; next() + // latches it so the newline after it is not suppressed as a continuation. + setterAssign bool + pendingSetterAssign bool } func New(src string) *Lexer { @@ -104,12 +118,35 @@ func (l *Lexer) Tokenize() []token.Token { // trailing-operator line continuation. The lexing itself is in lexToken. func (l *Lexer) next() token.Token { l.pendingBinary = false + l.pendingSetterAssign = false t := l.lexToken() l.prevType = t.Type l.prevBinary = l.pendingBinary + l.setterAssign = l.pendingSetterAssign + // Track def-name context across tokens. `def` arms defNamePending; the next + // IDENT (the method name) consumes it and arms defNameJustRead for exactly the + // following token, so a `=` hugging that name — and only there — is recognised + // as the setter terminator (`def x=`). Every other token clears both flags. + wasPending := l.defNamePending + l.defNameJustRead = false + l.defNamePending = false + switch { + case t.Type == token.DEF: + l.defNamePending = true + case t.Type == token.IDENT && wasPending: + l.defNameJustRead = true + } return t } +// regexAmbiguousAfterValueKeyword reports whether the previous token is one of +// the keywords (`super`, `yield`) that may stand as a bare value, so a following +// `/` is ambiguous between a division operator and a regexp-literal argument. +// The caller resolves it with the surrounding-space heuristic. +func (l *Lexer) regexAmbiguousAfterValueKeyword() bool { + return l.prevType == token.SUPER || l.prevType == token.YIELD +} + // isContinuationOp reports whether a line ending in token type t is incomplete, // so the trailing newline continues onto the next line rather than terminating // the statement. These are infix operators (plus comma and a trailing dot) that @@ -150,11 +187,17 @@ func (l *Lexer) lexToken() token.Token { switch { case c == 0: return mk(token.EOF, "") - case c == '/' && l.state == exprBegin && l.prevType != token.DEF: + case c == '/' && l.state == exprBegin && l.prevType != token.DEF && + !(l.regexAmbiguousAfterValueKeyword() && spaceBefore && l.peek2() == ' '): // At expression-begin position a '/' opens a regexp literal, not division // (the same disambiguation MRI uses via its lexer state). The one exception // is right after `def`, where `/` names the division-operator method // (`def /(other)`) rather than opening a pattern. + // + // `super`/`yield` may be used as bare values (`super / x`, `yield / 2`), + // where a `/` spaced on both sides is division, not a regexp argument. The + // `spaceBefore && spaceAfter` heuristic keeps `super /re/` (arg, no space + // after) a regexp while making `super / x` division — matching MRI. return l.lexRegexp(spaceBefore, line, col) case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): // %w[…] / %i[…] / %W[…] / %I[…] word- and symbol-array literals. @@ -198,7 +241,7 @@ func (l *Lexer) lexToken() token.Token { // Trailing-operator continuation: a line ending in an infix operator // (`a ||`, `x +`, a trailing comma, …) is incomplete and joins the next // line. `;` is an explicit terminator and is never suppressed this way. - if c == '\n' && isContinuationOp(l.prevType) { + if c == '\n' && isContinuationOp(l.prevType) && !l.setterAssign { return l.next() } // The ambiguous bitwise/shift operators `|`/`&`/`^`/`<<` continue a line @@ -413,6 +456,11 @@ func (l *Lexer) lexToken() token.Token { return mk(token.MATCH, "=~") } l.state = exprBegin + // A `=` hugging the def method name is the setter terminator (`def x=`); flag + // it so its trailing newline is not suppressed as a line continuation. + if l.defNameJustRead { + l.pendingSetterAssign = true + } return mk(token.ASSIGN, "=") case '!': if l.peek() == '=' { @@ -962,6 +1010,14 @@ func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token { l.advance() // closing '/' break } + if c == '#' && l.peek2() == '{' { + // An interpolation `#{…}` is copied verbatim into the source; a `/` (or a + // nested regexp `/…/`) inside it must not be mistaken for the closing + // delimiter, so consume the whole brace-balanced interpolation as-is. + src = append(src, l.advance(), l.advance()) // '#' '{' + src = l.copyInterpolation(src) + continue + } if c == '\\' { l.advance() esc := l.peek() @@ -994,6 +1050,67 @@ func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token { return token.Token{Type: token.REGEXP, Lit: string(src), Flags: string(flags), Line: line, Col: col, SpaceBefore: spaceBefore} } +// copyInterpolation copies a brace-balanced `#{…}` interpolation body verbatim +// onto dst, having already consumed the opening `#{`. It stops after the +// matching `}`. Strings (`"…"`, `'…'`) inside the body are copied whole so a `}` +// or `/` in them does not affect balancing; an unterminated body stops at EOF. +// Used by regexp lexing so a nested regexp `/…/` inside `#{…}` is not mistaken +// for the regexp's closing delimiter. +func (l *Lexer) copyInterpolation(dst []byte) []byte { + depth := 1 + for { + c := l.peek() + if c == 0 { + return dst + } + switch c { + case '{': + depth++ + dst = append(dst, l.advance()) + case '}': + depth-- + dst = append(dst, l.advance()) + if depth == 0 { + return dst + } + case '"', '\'': + dst = l.copyQuoted(dst, c) + case '\\': + dst = append(dst, l.advance()) + if l.peek() != 0 { + dst = append(dst, l.advance()) + } + default: + dst = append(dst, l.advance()) + } + } +} + +// copyQuoted copies a quoted string starting at the opening quote `q` (`"` or +// `'`) verbatim onto dst, including the closing quote, honouring backslash +// escapes. It is used inside copyInterpolation so braces inside a string do not +// affect interpolation brace-balancing. +func (l *Lexer) copyQuoted(dst []byte, q byte) []byte { + dst = append(dst, l.advance()) // opening quote + for { + c := l.peek() + if c == 0 { + return dst + } + if c == '\\' { + dst = append(dst, l.advance()) + if l.peek() != 0 { + dst = append(dst, l.advance()) + } + continue + } + dst = append(dst, l.advance()) + if c == q { + return dst + } + } +} + // charLiteralBegins reports whether a `?` may open a character literal here, // mirroring the regexp/percent rule: at expression-begin a value is expected, and // in command-argument position (just after a value, but with a space before the @@ -1073,8 +1190,30 @@ func (l *Lexer) lexCharLiteral(spaceBefore bool, line, col int) token.Token { val = "\x0c" case 'e': val = "\x1b" - case '0': + case 'x': + // `?\xHH` — a hex byte (one or two hex digits). Consume the digits so + // they are not lexed as a trailing integer. val = "\x00" + l.consumeHexEscape(2) + case 'u': + // `?\uHHHH` or `?\u{H…}` — a Unicode escape. Consume the payload. + val = "u" + l.consumeUnicodeEscape() + case 'C', 'M': + // Control/meta escapes: `?\C-a`, `?\M-x`, and combined `?\M-\C-a`. + // Consume the `-` and the (possibly itself-escaped) target character. + val = string(esc) + l.consumeControlMetaEscape() + case '0', '1', '2', '3', '4', '5', '6', '7': + // Octal escape `?\0`, `?\012` — consume up to two further octal digits. + val = "\x00" + for i := 0; i < 2; i++ { + if c := l.peek(); c >= '0' && c <= '7' { + l.advance() + } else { + break + } + } default: val = string(esc) } @@ -1091,6 +1230,60 @@ func (l *Lexer) lexCharLiteral(spaceBefore bool, line, col int) token.Token { return token.Token{Type: token.STRING, Lit: val, Line: line, Col: col, SpaceBefore: spaceBefore} } +// consumeHexEscape consumes up to max hexadecimal digits at the cursor, used to +// swallow the digits of a `\xHH` escape so they are not re-lexed as an integer. +func (l *Lexer) consumeHexEscape(max int) { + for i := 0; i < max; i++ { + if isHexDigit(l.peek()) { + l.advance() + } else { + break + } + } +} + +// consumeUnicodeEscape consumes a `\u` payload at the cursor: either `{H… H…}` +// (one or more space-separated code points) or exactly four hex digits. +func (l *Lexer) consumeUnicodeEscape() { + if l.peek() == '{' { + l.advance() + for l.peek() != '}' && l.peek() != 0 { + l.advance() + } + if l.peek() == '}' { + l.advance() + } + return + } + l.consumeHexEscape(4) +} + +// consumeControlMetaEscape consumes the remainder of a `\C-x` / `\M-x` control or +// meta escape at the cursor (the leading `\C`/`\M` is already consumed): an +// optional `-`, then the target — itself possibly a `\`-escape (`\M-\C-a`). +func (l *Lexer) consumeControlMetaEscape() { + if l.peek() == '-' { + l.advance() + } + if l.peek() == '\\' { + l.advance() + // A nested control/meta escape (`\M-\C-a`): consume its letter then recurse + // onto the final target character. + if c := l.peek(); c == 'C' || c == 'M' { + l.advance() + l.consumeControlMetaEscape() + return + } + } + if l.peek() != 0 { + l.advance() + } +} + +func isHexDigit(c byte) bool { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') +} + // percentDelimClose returns the closing delimiter for a %-literal opener: the // mate of a bracket pair, or the same character for a symmetric delimiter. func percentDelimClose(open byte) byte { @@ -1243,6 +1436,14 @@ func (l *Lexer) lexPercentRXS(spaceBefore bool, line, col int) token.Token { } continue } + if c == '#' && l.peek2() == '{' { + // An interpolation `#{…}` is copied verbatim; a string or nested literal + // inside it (which may contain the delimiter character) must not close the + // %-literal, so consume the whole brace-balanced interpolation as-is. + body = append(body, l.advance(), l.advance()) // '#' '{' + body = l.copyInterpolation(body) + continue + } if open != closing && c == open { depth++ } else if c == closing { diff --git a/parser.go b/parser.go index afdeecb..f188e1c 100644 --- a/parser.go +++ b/parser.go @@ -111,6 +111,11 @@ type Parser struct { // default (`def f(a = c(1), b = nil)`), so the comma separating parameters is // not mistaken for a masgn target separator (`c(1), b = nil`). noMasgn bool + // bracketDepth > 0 while parsing a parenthesised call-argument list or a hash + // literal, where newlines are insignificant. A `key:` whose value sits on the + // next line (`f(`⏎` k:`⏎` v)`) is then a continued pair, not a value-omitted + // shorthand; at top level (depth 0) a newline after `key:` ends the command. + bracketDepth int } // parseHook, when non-nil, runs at the start of Parse. It exists only so a @@ -259,6 +264,13 @@ var ( ) func (p *Parser) parseStatements(stop map[token.Type]bool) []ast.Node { + // A statement body is a fresh expression context: the masgn-suppression set + // while parsing an enclosing command-argument / parameter-default does not + // reach into a nested body (`p begin; a, b = z; end` is a real masgn). Clear it + // for the duration and restore on exit. + savedMasgn := p.noMasgn + p.noMasgn = false + defer func() { p.noMasgn = savedMasgn }() var body []ast.Node for { p.skipNewlines() @@ -503,24 +515,36 @@ func (p *Parser) parseDef() ast.Node { var params []string var defaults []ast.Node var kwParams []ast.KwParam + var prepends []ast.Node var kwRest, blockParam string forward := false splat := -1 if p.accept(token.LPAREN) { - params, defaults, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.RPAREN) + params, defaults, prepends, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.RPAREN) p.expect(token.RPAREN) - } else if (p.is(token.IDENT) || p.is(token.LABEL) || p.is(token.AMPER) || p.is(token.DOTDOTDOT)) && !p.is(token.NEWLINE) { - // paren-less params: def foo a, b / def foo a:, b: 2 / def foo &blk - params, defaults, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.NEWLINE) + } else if (p.is(token.IDENT) || p.is(token.LABEL) || p.is(token.AMPER) || p.is(token.DOTDOTDOT) || + p.is(token.STAR) || p.is(token.POW) || p.is(token.LPAREN)) && !p.is(token.NEWLINE) { + // paren-less params: def foo a, b / def foo a:, b: 2 / def foo &blk / + // def foo *rest / def foo **opts / def foo (a, b), c + params, defaults, prepends, splat, kwParams, kwRest, blockParam, forward = p.parseDefParams(token.NEWLINE) } // Endless method definition: def name(params) = expr (no body/end). if p.accept(token.ASSIGN) { body := []ast.Node{p.parseExprOrAssign()} + if len(prepends) > 0 { + body = append(prepends, body...) + } p.popScope() return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Forward: forward, Body: body} } // A method body may carry rescue/else/ensure clauses without an explicit begin. body := p.parseBodyWithRescue() + // A destructuring parameter `(a, b)` expands to a multiple-assignment from a + // synthetic positional, prepended to the body (the same shape parseBlockParams + // uses for block destructuring). + if len(prepends) > 0 { + body = append(prepends, body...) + } p.popScope() p.expect(token.END) return &ast.MethodDef{Name: name, Params: params, Defaults: defaults, SplatIndex: splat, KwParams: kwParams, KwRest: kwRest, BlockParam: blockParam, Singleton: singleton, Recv: recv, Forward: forward, Body: body} @@ -568,9 +592,15 @@ func (p *Parser) parseDefName() (string, bool) { return "`", true } // A keyword used as a method name: def do / def then / def in / def class … - // Ruby permits any reserved word in def-name position. + // Ruby permits any reserved word in def-name position, including the setter + // form `def ensure=(v)` where a `=` hugs the keyword name. if _, isKeyword := token.Keywords[p.cur().Lit]; isKeyword { - return p.advance().Lit, true + name := p.advance().Lit + if p.is(token.ASSIGN) && !p.cur().SpaceBefore { + p.advance() + name += "=" + } + return name, true } return "", false } @@ -579,8 +609,9 @@ func (p *Parser) parseDefName() (string, bool) { // default`. Each parameter is declared before its (and later) defaults are // parsed, so a default may reference earlier parameters. defaults is parallel to // params, nil for a required parameter. -func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []ast.Node, splat int, kwParams []ast.KwParam, kwRest, blockParam string, forward bool) { +func (p *Parser) parseDefParams(until token.Type) (params []string, defaults, prepends []ast.Node, splat int, kwParams []ast.KwParam, kwRest, blockParam string, forward bool) { splat = -1 + group := 0 // A parenthesised parameter list may span several lines, with newlines after // the open paren and around the separating commas; a paren-less list (until == // NEWLINE) must not skip newlines, as one terminates it. @@ -589,7 +620,7 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a p.skipNewlines() } if p.is(until) || p.is(token.NEWLINE) { - return params, defaults, splat, kwParams, kwRest, blockParam, forward + return params, defaults, prepends, splat, kwParams, kwRest, blockParam, forward } for { if paren { @@ -598,6 +629,25 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a break } } + if p.is(token.LPAREN) { // destructuring positional param: `((a, b), c)` + outer, chained := p.parseDestructureParam(&group) + // The synthetic positional carrying the destructured value is recorded as + // the parameter (so arity is correct); the unpacking MultiAssigns run at + // the top of the body. Inner (nested) unpacks come first so each + // synthetic is bound before the outer unpack reads it. + params = append(params, outer.Values[0].(*ast.VarRef).Name) + defaults = append(defaults, nil) + // The outer unpack binds the inner synthetics first, then the chained + // inner unpacks read them. + prepends = append(prepends, outer) + for _, c := range chained { + prepends = append(prepends, c) + } + if !p.accept(token.COMMA) { + break + } + continue + } if p.accept(token.DOTDOTDOT) { // `...` argument-forwarding param (always last) forward = true break @@ -682,7 +732,50 @@ func (p *Parser) parseDefParams(until token.Type) (params []string, defaults []a if paren { p.skipNewlines() // tolerate a newline before the close paren } - return params, defaults, splat, kwParams, kwRest, blockParam, forward + return params, defaults, prepends, splat, kwParams, kwRest, blockParam, forward +} + +// parseDestructureParam parses a parenthesised destructuring positional +// parameter — `(a, b)`, `((a, b), c)`, `(a, *b, c)` — returning a MultiAssign +// that unpacks a synthetic positional ("(0)", "(1)", …) into the named locals, +// to be prepended to the method body. It mirrors the block-destructuring shape +// in parseBlockParams and supports one level of further nesting. +func (p *Parser) parseDestructureParam(group *int) (outer *ast.MultiAssign, chained []*ast.MultiAssign) { + p.expect(token.LPAREN) + var names []string + gsplat := -1 + for { + switch { + case p.accept(token.STAR): + gsplat = len(names) + if p.is(token.IDENT) { + n := p.advance().Lit + names = append(names, n) + p.declareLocal(n) + } else { + names = append(names, "*") + } + case p.is(token.LPAREN): + // A further-nested destructure: this outer unpack binds it into its own + // synthetic local; a chained MultiAssign then unpacks that synthetic. + inner, innerChained := p.parseDestructureParam(group) + names = append(names, inner.Values[0].(*ast.VarRef).Name) + chained = append(chained, inner) + chained = append(chained, innerChained...) + default: + n := p.expect(token.IDENT).Lit + names = append(names, n) + p.declareLocal(n) + } + if !p.accept(token.COMMA) { + break + } + } + p.expect(token.RPAREN) + syn := "(" + strconv.Itoa(*group) + ")" + *group++ + p.declareLocal(syn) + return &ast.MultiAssign{Names: names, SplatIndex: gsplat, Values: []ast.Node{&ast.VarRef{Name: syn}}}, chained } // parseParamDefault parses a parameter's default-value expression with masgn @@ -696,10 +789,27 @@ func (p *Parser) parseParamDefault() ast.Node { return def } +// parseKwArgValue parses the value of a `key: value` keyword argument with masgn +// detection suppressed, so an assignment value does not swallow the comma before +// the next argument (`f(a: x = 1, b: y = 2)` is two pairs, not one masgn value). +func (p *Parser) parseKwArgValue() ast.Node { + saved := p.noMasgn + p.noMasgn = true + v := p.parseExprOrAssign() + p.noMasgn = saved + return v +} + // parseCond parses an if/unless/while/until condition, where the low-precedence // keyword operators `and`/`or`/`not` are permitted (`if a and b`, `while x or // y`, `unless not done`). -func (p *Parser) parseCond() ast.Node { return p.parseKeywordLogical() } +func (p *Parser) parseCond() ast.Node { + // A condition may itself be a one-line pattern match: `if node in Foo[...]`, + // `while x => p`. Wrap the logical expression so a trailing `in`/`=>` pattern + // is consumed here too, not only at statement level. (A `case`'s `in` uses a + // separate path and never reaches parseCond.) + return p.parseOneLineMatch(p.parseKeywordLogical()) +} func (p *Parser) parseIf() ast.Node { p.expect(token.IF) @@ -764,6 +874,30 @@ func (p *Parser) parseUntil() ast.Node { return &ast.While{Cond: not(cond), Body: body} } +// parseFor parses `for VAR[, VAR…] in ITER [do] ... end`. The loop variables are +// plain names (one or more, comma-separated) that — unlike block parameters — +// are declared in the enclosing scope and outlive the loop, so they are recorded +// as locals here. The iterator expression is parsed with `do…end` attachment +// suppressed so a trailing `do` belongs to the loop, not to a call within it. +func (p *Parser) parseFor() ast.Node { + p.expect(token.FOR) + var vars []string + for { + name := p.expect(token.IDENT).Lit + vars = append(vars, name) + p.declareLocal(name) + if !p.accept(token.COMMA) { + break + } + } + p.expect(token.IN) + iter := p.parseLoopCond() + p.accept(token.DO) + body := p.parseStatements(bodyEnd) + p.expect(token.END) + return &ast.For{Vars: vars, Iter: iter, Body: body} +} + func (p *Parser) parseReturn() ast.Node { p.expect(token.RETURN) // A value-less `return` ends at a terminator, a body/block close, or a @@ -1198,19 +1332,26 @@ func (p *Parser) parsePatternValue() ast.Node { // with bracket delimiters (`Const[key:]`, deconstruct_keys), which MRI accepts. func (p *Parser) parseArrayPattern(constName ast.Node) ast.Pattern { p.expect(token.LBRACKET) + // A bracket pattern body may span several lines (`Const[`⏎` a: 1,`⏎`]`), so + // newlines after the `[` and around the separating commas are insignificant. + p.skipNewlines() if p.is(token.LABEL) || p.is(token.POW) { hp := p.parseHashPatternBody(constName, token.RBRACKET) + p.skipNewlines() p.expect(token.RBRACKET) return hp } var elems []arrayElem if !p.accept(token.RBRACKET) { elems = append(elems, p.parseArrayPatternElem()) + p.skipNewlines() for p.accept(token.COMMA) { + p.skipNewlines() if p.is(token.RBRACKET) { // trailing comma break } elems = append(elems, p.parseArrayPatternElem()) + p.skipNewlines() } p.expect(token.RBRACKET) } @@ -1758,6 +1899,12 @@ func (p *Parser) parseExprOrAssign() ast.Node { // 2, 3]` and `a = *list, y` ≡ `a = [*list, y]`, matching MRI. func (p *Parser) parseAssignRhs() ast.Node { first := p.parseRhsElem() + // While masgn is suppressed (a parameter default or a keyword-argument value), + // a comma ends this value rather than gathering an implicit array RHS, so + // `f(a: x = 1, b: 2)` keeps `b: 2` as the next argument. + if p.noMasgn { + return first + } if !p.is(token.COMMA) { // A bare leading `*splat` with no comma still yields a one-element array // (`a = *list` ≡ `a = [*list]`), as MRI does. @@ -1832,7 +1979,10 @@ func (p *Parser) parseRange() ast.Node { p.advance() var hi ast.Node // endless when nothing can follow the dots if !rangeHiEnds[p.cur().Type] { - hi = p.parseBinary(0) + // The endpoint may be an assignment (`a..b = c`) or op-assignment + // (`@i...@i += n`), which MRI parses as the range's high bound, so feed + // the parsed expression through the inline-assignment handler. + hi = p.maybeInlineAssign(p.parseBinary(0)) } return &ast.RangeLit{Lo: left, Hi: hi, Exclusive: excl} } @@ -2242,6 +2392,11 @@ func (p *Parser) parseHashLiteral() ast.Node { // `name: value` — the label is sugar for a symbol key. name := p.advance().Lit k = &ast.SymbolLit{Name: name} + // A value on the next line (`{`⏎` k:`⏎` v }`) is a continued pair: + // skip the newline unless a separator/closing token follows (shorthand). + if p.is(token.NEWLINE) && !p.firstSignificantIs(token.COMMA) && !p.firstSignificantIs(token.RBRACE) { + p.skipNewlines() + } if p.is(token.COMMA) || p.is(token.RBRACE) || p.is(token.NEWLINE) { v = p.barewordValue(name) // `{x:}` shorthand == `{x: x}` } else { @@ -2360,11 +2515,22 @@ func (p *Parser) parseBlockRest(stop map[token.Type]bool, end token.Type, withRe // `foo {``|x| … }`, `do``|a, b| … end`. Skip the intervening newlines // before the leading `|` (a body statement can never begin with a bare `|`, so // this is unambiguous). The newlines are no-ops for the body either way. - if p.is(token.NEWLINE) && p.firstSignificantIs(token.PIPE) { + if p.is(token.NEWLINE) && (p.firstSignificantIs(token.PIPE) || p.firstSignificantIs(token.OROR)) { p.skipNewlines() } - if p.accept(token.PIPE) { + // An empty parameter list `||` is lexed as a single OROR token (`{ || body }`). + if p.accept(token.OROR) { + p.scope().explicitParams = true + } else if p.accept(token.PIPE) { params, defaults, prepends, splat, blockParam = p.parseBlockParams(token.PIPE) + // Block-local variables follow a `;`: `|a, b; x, y|`. The lexer maps `;` to a + // NEWLINE token, so a NEWLINE here (before the closing `|`) opens the + // block-local list. They are ordinary locals of the block scope — declared so + // the body sees them, but carried no further than the parameter list. + if p.is(token.NEWLINE) { + p.advance() + p.parseBlockLocals() + } p.expect(token.PIPE) p.scope().explicitParams = true } @@ -2391,6 +2557,20 @@ func (p *Parser) parseBlockRest(stop map[token.Type]bool, end token.Type, withRe return &ast.Block{Params: params, Defaults: defaults, SplatIndex: splat, BlockParam: blockParam, Body: body} } +// parseBlockLocals parses the block-local variable list after the `;` in a block +// parameter list (`|a, b; x, y|`). Each name is declared in the current block +// scope so the body resolves it as a local; the list is otherwise discarded +// since block-locals carry no arity or default. +func (p *Parser) parseBlockLocals() { + for { + name := p.expect(token.IDENT).Lit + p.declareLocal(name) + if !p.accept(token.COMMA) { + break + } + } +} + // parseBlockParams parses a block's parameter list (the `|...|` form for brace/do // blocks and the `(...)` form for stabby lambdas), where each parameter is either // a plain name, an optional `name = default` param, a top-level `*rest` splat, or @@ -2720,6 +2900,8 @@ func (p *Parser) parsePrimary() ast.Node { return p.parseWhile() case token.UNTIL: return p.parseUntil() + case token.FOR: + return p.parseFor() } return p.fail("unexpected token %q (%s)", t.Lit, t.Type) } @@ -2768,9 +2950,23 @@ func (p *Parser) parseIdentExpr() ast.Node { return call } - // Known local variable → read. + // Known local variable → read. But a bareword in scope is still a method call + // when it is immediately followed by a construct a local variable cannot take: + // a `do` block (`fork = false; pid = fork do … end`) or an unambiguous + // command argument (`type = 1; type 'X = Y'` is `type('X = Y')`). A brace block + // binds tighter and is handled by the postfix chain. if p.is(token.IDENT) && p.isLocal(name) { p.advance() + if p.is(token.DO) && !p.noDo { + return &ast.Call{Name: name, Block: p.parseDoBlock()} + } + if p.localCommandArgFollows() { + call := &ast.Call{Name: name, Args: p.parseCommandArgs()} + if p.is(token.DO) && !p.noDo { + call.Block = p.parseDoBlock() + } + return call + } return &ast.VarRef{Name: name} } @@ -2791,8 +2987,10 @@ func (p *Parser) parseIdentExpr() ast.Node { return &ast.Call{Name: name, Args: p.parseCommandArgs()} } // Bare `it` (no receiver, no args) inside a param-less block is the implicit - // single parameter (Ruby 3.4). With args/parens it stays a method call. - if name == "it" { + // single parameter (Ruby 3.4). With args/parens — or a block, as in RSpec's + // `it { … }` / `it do … end` — it stays a method call taking that block, not + // the implicit parameter followed by a stray block. + if name == "it" && !p.is(token.LBRACE) && !(p.is(token.DO) && !p.noDo) { if s := p.implicitParamScope(); s != nil { s.usedIt = true return &ast.VarRef{Name: name} @@ -2818,18 +3016,44 @@ func isHuggingString(t token.Token) bool { } // constCommandArgFollows reports whether a constant is immediately followed by a -// space-separated literal value, making it a conversion-style command call -// (`BigDecimal "0.01"`, `Integer "42"`, `Float 1`). It is deliberately narrow — -// only a literal string/number/symbol argument — so an ordinary `Foo` reference -// next to other tokens (a binary operator, `.method`, `[`, a newline) is not -// mistaken for a call. +// space-separated value, making it a conversion-/command-style call: a literal +// (`BigDecimal "0.01"`, `Integer "42"`, `Float 1`) or another value-starting +// token (`raise ArgumentError urn.inspect`, `X y`). It stays deliberately narrow +// — only unambiguous argument starts — so an ordinary `Foo` reference next to an +// operator, `.method`, `::`, `[`, or a newline is not mistaken for a call. func (p *Parser) constCommandArgFollows() bool { t := p.cur() if !t.SpaceBefore { return false } switch t.Type { - case token.STRING, token.STRBEG, token.INT, token.FLOAT, token.SYMBOL: + case token.STRING, token.STRBEG, token.INT, token.FLOAT, token.SYMBOL, + token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, + token.TRUE, token.FALSE, token.NIL, token.SELF, + token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING: + return true + } + return false +} + +// localCommandArgFollows decides whether a bareword that is a known local should +// nonetheless be read as a command call because an unambiguous argument follows +// (`type = 1; type 'X'` is `type('X')`). It is deliberately narrower than +// canStartCommandArg: for an in-scope name the sign/splat/block-pass operators +// stay binary (`x = 5; x -1` is `x - 1`, not `x(-1)`), and a bare `(`/`[` is an +// index/group on the value, so only space-separated literal/value-starting +// arguments — a string, number, symbol, another bareword, a constant, an ivar, +// etc. — promote the local back to a call. +func (p *Parser) localCommandArgFollows() bool { + t := p.cur() + if !t.SpaceBefore { + return false + } + switch t.Type { + case token.STRING, token.STRBEG, token.INT, token.FLOAT, token.SYMBOL, + token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, + token.TRUE, token.FALSE, token.NIL, token.SELF, + token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING, token.LABEL: return true } return false @@ -2847,11 +3071,12 @@ func (p *Parser) canStartCommandArg() bool { case token.INT, token.FLOAT, token.STRING, token.STRBEG, token.SYMBOL, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, token.TRUE, token.FALSE, token.NIL, token.SELF, token.BANG, token.TILDE, token.LPAREN, token.LBRACKET, token.ARROW, token.WORDS, token.SYMBOLS, token.REGEXP, token.XSTRING, - token.BEGIN, token.CASE, token.DEF, token.SUPER: + token.BEGIN, token.CASE, token.DEF, token.SUPER, token.YIELD: // Value-producing keywords: `p begin; 1; end`, `p case x; when 1; 2; end`, // `def` (which evaluates to the method's name symbol, as in - // `module_function def foo; end` / `private def bar; end`), and `super` - // (`number_to_currency super`, `Request.new super, url_helpers`). + // `module_function def foo; end` / `private def bar; end`), `super` + // (`number_to_currency super`, `Request.new super, url_helpers`), and `yield` + // (`raise yield('x')`, `foo yield(1)` — the block's value as an argument). return true case token.LABEL: // Keyword/hash argument without parens: `render json: x`, `delegate to: :c`. @@ -2881,6 +3106,13 @@ func (p *Parser) parseCommandArgs() []ast.Node { // binds tighter and always attaches to the nearest call. saved := p.noDo p.noDo = true + // In a paren-less argument list the comma separates arguments, so multiple- + // assignment detection must be off: `assert_equal a(1), tag = b(2)` is two + // arguments (the second an assignment), not an mlhs `a(1), tag = …`. A single + // argument that is itself an assignment (`f tag = y`) still parses, and an + // assignment value stops at the comma. Mirrors parameter-default handling. + savedMasgn := p.noMasgn + p.noMasgn = true var args []ast.Node var kw *ast.HashLit p.parseOneCallArg(&args, &kw) @@ -2888,6 +3120,7 @@ func (p *Parser) parseCommandArgs() []ast.Node { p.skipNewlines() p.parseOneCallArg(&args, &kw) } + p.noMasgn = savedMasgn p.noDo = saved if kw != nil { args = append(args, kw) @@ -2896,6 +3129,8 @@ func (p *Parser) parseCommandArgs() []ast.Node { } func (p *Parser) parseCallArgs(until token.Type) []ast.Node { + p.bracketDepth++ + defer func() { p.bracketDepth-- }() var args []ast.Node var kw *ast.HashLit p.skipNewlines() @@ -2954,6 +3189,16 @@ func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { if p.is(token.LABEL) { name := p.advance().Lit key := &ast.SymbolLit{Name: name} + // Inside a parenthesised argument list or hash, a `key:` whose value is on + // the next line is a continued pair: skip the intervening newline(s) so the + // value is parsed, unless what follows is itself a separator/closing token + // (then it is a value-omitted shorthand). At top level a newline ends the + // command, so this skipping is gated on bracketDepth. + if p.bracketDepth > 0 && p.is(token.NEWLINE) && !p.firstSignificantIs(token.COMMA) && + !p.firstSignificantIs(token.RPAREN) && !p.firstSignificantIs(token.RBRACKET) && + !p.firstSignificantIs(token.RBRACE) { + p.skipNewlines() + } // Value-omitted shorthand (Ruby 3.4): `foo(format:, name:)` means // `foo(format: format, name: name)` — when the label is not followed by a // value, the value is the same-named local or method call. @@ -2961,7 +3206,11 @@ func (p *Parser) parseOneCallArg(args *[]ast.Node, kw **ast.HashLit) { p.addKwPair(kw, key, p.barewordValue(name)) return } - p.addKwPair(kw, key, p.parseExprOrAssign()) + // A keyword value that is itself an assignment must not swallow the comma + // separating the next argument: `f(a: x = 1, b: y = 2)` is two keyword + // pairs, not `a: (x = (1, b: y = 2))`. Suppress masgn detection (as a def + // parameter default does) so the `,` ends this value. + p.addKwPair(kw, key, p.parseKwArgValue()) return } if p.accept(token.STAR) { diff --git a/round5_test.go b/round5_test.go new file mode 100644 index 0000000..ddd5b47 --- /dev/null +++ b/round5_test.go @@ -0,0 +1,367 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// mustParseR5 parses src, failing the test on a parse error. It is the shared +// success oracle for the Round-5 features, each of which is MRI-valid (verified +// with `ruby -c` against MRI 4.0.x). +func mustParseR5(t *testing.T, src string) *ast.Program { + t.Helper() + prog, err := parser.Parse(src) + if err != nil { + t.Fatalf("Parse(%q): %v", src, err) + } + return prog +} + +// mustFailR5 asserts that src is rejected, used for the constructs MRI itself +// rejects so the parser's matching behaviour is pinned. +func mustFailR5(t *testing.T, src string) { + t.Helper() + if _, err := parser.Parse(src); err == nil { + t.Fatalf("Parse(%q): expected a parse error, got none", src) + } +} + +// TestR5ParenlessDefParams covers a method definition whose parameters have no +// surrounding parentheses, including leading *splat / **double-splat and a +// trailing &block (`def f a, b`, `def initialize *names`, `def g *a, &b`). +func TestR5ParenlessDefParams(t *testing.T) { + for _, src := range []string{ + "def initialize *argnames\nend\n", + "def f a, b\nend\n", + "def g *a, &b\nend\n", + "def h a, *b, c:\nend\n", + "def k a: 1, b: 2\nend\n", + "def dd **opts\nend\n", + "def m a, b = 2, *c, d:, e: 5, **f, &g\nend\n", + } { + mustParseR5(t, src) + } +} + +// TestR5DestructuringDefParams covers a parenthesised destructuring positional +// parameter in a def list, including a further-nested group. +func TestR5DestructuringDefParams(t *testing.T) { + for _, src := range []string{ + "def f((a, b), c)\n a + b + c\nend\n", + "def make_tmpname((prefix, suffix), n)\n [prefix, suffix, n]\nend\n", + "def g(a, (b, c), *d)\n [a, b, c, d]\nend\n", + "def h((a, (b, c)), d)\n [a, b, c, d]\nend\n", + "def i((a, *b), c)\n [a, b, c]\nend\n", + "def j((a, b), (c, d))\n [a, b, c, d]\nend\n", // two groups, first not last + "def k((a, *), c)\n [a, c]\nend\n", // anonymous splat inside a group + "def m((a, b)) = a + b\n", // endless method with a destructure + } { + mustParseR5(t, src) + } + // A paren-less destructuring head is a SyntaxError in MRI too. + mustFailR5(t, "def f (a, b), c\n a\nend\n") +} + +// TestR5DestructuringDefBody verifies a destructuring parameter expands to a +// leading multiple-assignment prepended to the method body. +func TestR5DestructuringDefBody(t *testing.T) { + prog := mustParseR5(t, "def f((a, b), c)\n a\nend\n") + md := prog.Body[0].(*ast.MethodDef) + if len(md.Body) == 0 { + t.Fatalf("empty body") + } + ma, ok := md.Body[0].(*ast.MultiAssign) + if !ok { + t.Fatalf("body[0] = %T, want *ast.MultiAssign (the destructure unpack)", md.Body[0]) + } + if len(ma.Names) != 2 || ma.Names[0] != "a" || ma.Names[1] != "b" { + t.Fatalf("unpack names = %v, want [a b]", ma.Names) + } +} + +// TestR5CharLiteralEscapes covers hex / unicode / octal / control / meta escapes +// in a `?x` character literal (the simple escapes were already supported). +func TestR5CharLiteralEscapes(t *testing.T) { + for _, src := range []string{ + "x = ?\\x00\n", + "x = ?\\xff\n", + "x = ?\\u00e9\n", + "x = ?\\u{1f600}\n", + "x = ?\\C-a\n", + "x = ?\\M-x\n", + "x = ?\\M-\\C-a\n", + "x = ?\\012\n", + "x.split(?\\x00)\n", + "x = ?\\s\n", + "x = ?a\n", + } { + mustParseR5(t, src) + } +} + +// TestR5BlockParams covers an empty `||` parameter list and block-local +// variables declared after a `;` in the parameter list. +func TestR5BlockParams(t *testing.T) { + for _, src := range []string{ + "x { || 'yay' }\n", + "lambda { || x }\n", + "proc { || }\n", + "x { |a; y, z| a }\n", + "x { |a; y| a }\n", + "x { |; y| y }\n", + "x.each { |a, b; tmp| tmp }\n", + "x do |a; t| t end\n", + } { + mustParseR5(t, src) + } +} + +// TestR5RegexInIndexAndInterp covers a regexp literal in index/value position and +// a nested regexp inside an interpolation inside an outer regexp. +func TestR5RegexInIndexAndInterp(t *testing.T) { + for _, src := range []string{ + "x[/\\A\\s*/]\n", + "obj[/re/]\n", + "h[/re/] = v\n", + "x[/\\A\\s*/, 1]\n", + "x = /^#{x[/\\A\\s*/]}/\n", + "x = /^#{ scan(/^\\s*/).min }/\n", + "x = /^#{ scan(/^\\s*/).min_by { |l| l.size } }/\n", + "%r\"^#{Regexp.escape 'import \"x\"'}\"\n", + } { + mustParseR5(t, src) + } +} + +// TestR5ControlFlowCommandArg covers a paren-less command whose argument is a +// `yield(...)` (or bare `yield`) control-flow call. +func TestR5ControlFlowCommandArg(t *testing.T) { + for _, src := range []string{ + "raise yield('x')\n", + "raise yield\n", + "foo yield(1)\n", + "puts yield\n", + "raise super(x)\n", + } { + mustParseR5(t, src) + } +} + +// TestR5ItMethodCall covers RSpec's `it { … }` / `it do … end`: a bare `it` +// taking a block is a method call, not the Ruby-3.4 implicit block parameter. +func TestR5ItMethodCall(t *testing.T) { + for _, src := range []string{ + "describe Foo do\n it { is_expected.to be_versionable }\nend\n", + "describe Foo do\n it do\n expect(1).to eq 1\n end\nend\n", + } { + prog := mustParseR5(t, src) + // The inner `it` is a Call carrying a block. + outer := prog.Body[0].(*ast.Call).Block + inner := outer.Body[0] + c, ok := inner.(*ast.Call) + if !ok || c.Name != "it" || c.Block == nil { + t.Fatalf("inner it = %T, want a *ast.Call named it with a block", inner) + } + } + // Bare `it` with no block is still the implicit parameter. + prog := mustParseR5(t, "[1].map { it * 2 }\n") + blk := prog.Body[0].(*ast.Call).Block + if len(blk.Params) != 1 || blk.Params[0] != "it" { + t.Fatalf("implicit it params = %v, want [it]", blk.Params) + } +} + +// TestR5LocalAndConstCommandArg covers a bareword that is a known local, or a +// constant, promoted back to a method call by a following command argument +// (`type = 1; type 'X'` → `type('X')`; `raise ArgumentError urn.inspect`). +func TestR5LocalAndConstCommandArg(t *testing.T) { + for _, src := range []string{ + "type = 1\ntype 'X = Y'\n", + "def f(type)\n type 'X = Y'\nend\n", + "fork = false\npid = fork do\n x\nend\n", + "raise ArgumentError urn.inspect unless match\n", + "X y, z\n", + "type = 1\ntype x do\n y\nend\n", // local promoted to a call taking a do block + } { + mustParseR5(t, src) + } + // Operators on an in-scope local stay binary, and a plain constant stays a + // reference — neither is mis-promoted to a call. + prog := mustParseR5(t, "x = 5\nx - 1\n") + if _, ok := prog.Body[1].(*ast.BinaryExpr); !ok { + t.Fatalf("x - 1 = %T, want *ast.BinaryExpr", prog.Body[1]) + } + prog = mustParseR5(t, "y = Foo\n") + if _, ok := prog.Body[0].(*ast.Assign).Value.(*ast.ConstRef); !ok { + t.Fatalf("Foo = %T, want *ast.ConstRef", prog.Body[0].(*ast.Assign).Value) + } +} + +// TestR5KeywordAndAssignArgs covers a keyword/assignment value that must not +// swallow the following argument, and command-argument multiple-assignment +// suppression (`f a(1), b = c`, `assert_equal a, tag = b`). +func TestR5KeywordAndAssignArgs(t *testing.T) { + for _, src := range []string{ + "f a: x = 1, b: y = 2\n", + "f(a: x = 1, b: y = 2)\n", + "assert_equal tags(:g), tag = posts(:w).tags.first\n", + "f a(1), b = c\n", + "f \\\n title: title = \"x\",\n body: body = \"y\"\n", + } { + mustParseR5(t, src) + } + // A real statement-level masgn still gathers its RHS array. + prog := mustParseR5(t, "a, b = 1, 2\n") + if _, ok := prog.Body[0].(*ast.MultiAssign); !ok { + t.Fatalf("a, b = 1, 2 = %T, want *ast.MultiAssign", prog.Body[0]) + } + // A masgn inside a nested body (command-arg begin) is not suppressed. + mustParseR5(t, "p begin\n a, b = z\nend\n") +} + +// TestR5LabelValueOnNextLine covers a `key:` whose value sits on the next line +// inside a parenthesised call or a hash literal, distinguished from the +// value-omitted shorthand. +func TestR5LabelValueOnNextLine(t *testing.T) { + for _, src := range []string{ + "f(\n k:\n v,\n)\n", + "{\n k:\n Foo::Bar.new(1),\n}\n", + "f(a:, b:)\n", + "f(\n a:,\n b:,\n)\n", + "{ a:, b: }\n", + } { + mustParseR5(t, src) + } +} + +// TestR5SetterDefNewlineBody covers a paren-less setter `def x=` whose body +// begins on the next line (the setter `=` must not continue the line), and a +// setter defined with a keyword name (`def ensure=(v)`). +func TestR5SetterDefNewlineBody(t *testing.T) { + for _, src := range []string{ + "def x=\n foo do\n end\nend\n", + "def test_session_auth=\n assert_raise(X) do\n y\n end\nend\n", + "def x= y\n y\nend\n", + "def x=(v)\n v\nend\n", + "def ensure=(v)\n v\nend\n", + "def class=(v)\n v\nend\n", + } { + mustParseR5(t, src) + } + // A hugging `=` outside a def is ordinary assignment and still continues a + // line onto the next. + mustParseR5(t, "x=\n5\n") + mustParseR5(t, "h = {}\nh[:a]=\n5\n") +} + +// TestR5ForLoop covers the `for VAR[, VAR…] in ITER [do] … end` loop, recording +// the loop variables and iterator in a For node. +func TestR5ForLoop(t *testing.T) { + prog := mustParseR5(t, "for a, b in pairs\n a\nend\n") + f, ok := prog.Body[0].(*ast.For) + if !ok { + t.Fatalf("body[0] = %T, want *ast.For", prog.Body[0]) + } + if len(f.Vars) != 2 || f.Vars[0] != "a" || f.Vars[1] != "b" { + t.Fatalf("for vars = %v, want [a b]", f.Vars) + } + for _, src := range []string{ + "for i in 1..100\n x\nend\n", + "for i in [1, 2] do\n x\nend\n", + "for x in foo.bar do\n z\nend\n", + "for i in 1..3\n y\nend\np i\n", + } { + mustParseR5(t, src) + } +} + +// TestR5SuperYieldDivision covers `super`/`yield` used as bare values with a +// division operator (`super / x`), distinguished from a regexp argument +// (`super /re/`) by the surrounding-space heuristic. +func TestR5SuperYieldDivision(t *testing.T) { + for _, src := range []string{ + "def f\n super / x\nend\n", + "super / @step_size\n", + "yield / 2\n", + "super /foo/\n", + "yield /foo/\n", + } { + mustParseR5(t, src) + } +} + +// TestR5PatternMatchCondAndMultiline covers a one-line pattern match in a +// condition (`if node in Foo[...]`) and a bracket pattern body spanning +// newlines. +func TestR5PatternMatchCondAndMultiline(t *testing.T) { + for _, src := range []string{ + "if node in Foo[a: 1]\n x\nend\n", + "if node in Foo[\n a: 1\n]\n x\nend\n", + "n in Foo[\n a: 1\n]\n", + "node in Foo[\n name: :x,\n args: Bar[a: [b]]\n]\n", + "x in [\n 1,\n 2\n]\n", + } { + mustParseR5(t, src) + } +} + +// TestR5InterpolationLexerEdges exercises the verbatim-copy of an interpolation +// inside a regexp / %-literal: nested braces, a `'`/`"` string holding the +// delimiter, backslash escapes, and unterminated forms (which the parser +// surfaces as an error rather than crashing). +func TestR5InterpolationLexerEdges(t *testing.T) { + for _, src := range []string{ + "x = /a#{ {b: 1} }/\n", // nested braces in interpolation + "x = /a#{ \"x/y\" }/\n", // double-quoted string holding the delimiter + "x = /a#{ 'x/y' }/\n", // single-quoted string holding the delimiter + "x = /a#{ \"x\\\"y\" }b/\n", // escaped quote inside the string + "x = /a#{ \"x\\\" }/\n", // backslash escape inside the interpolation string + "x = /a#{ b\\\nc }/\n", // backslash escape inside the interpolation itself + "%r\"#{ \"a/b\" }\"\n", // %r with a delimiter-bearing string in interp + "x[/\\xff/]\n", // hex escape consumed fully in a regexp index + } { + mustParseR5(t, src) + } + // Unterminated interpolation / quoted string inside a regexp must not crash: + // the lexer copy-loops stop at EOF and the parser still produces a result + // (the literal is taken as terminated at EOF, matching the bare-`/` behaviour). + for _, src := range []string{ + "x = /a#{ \"unterminated\n", + "x = /a#{ 'unterminated\n", + "x = /a#{ unterminated\n", + "x = /a#{ \"x\\", + } { + if _, err := parser.Parse(src); err != nil { + // A reported error is acceptable too; the point is no panic. + _ = err + } + } +} + +// TestR5CharEscapeShortForms exercises the char-escape consumers on truncated +// input: a `\x` with fewer than two hex digits and a bare `\u{` without a close. +func TestR5CharEscapeShortForms(t *testing.T) { + for _, src := range []string{ + "x = ?\\x9\n", // single hex digit (consumeHexEscape stops early) + "x = ?\\0\n", // single octal digit + "x = ?\\C-\\M-a\n", + } { + mustParseR5(t, src) + } +} + +// TestR5RangeEndpointAssign covers an assignment / op-assignment as a range's +// high bound (`a..b = c`, `@i...@i += n`), which MRI parses as the endpoint. +func TestR5RangeEndpointAssign(t *testing.T) { + for _, src := range []string{ + "x = (@a...@a += b.size)\n", + "self << (@bind_index...@bind_index += binds.size).map(&block).join(\", \")\n", + "x = (a..b = c)\n", + "x = (1..5)\n", + "x = (1..)\n", + } { + mustParseR5(t, src) + } +} diff --git a/token/token.go b/token/token.go index f7396f7..d7d26a2 100644 --- a/token/token.go +++ b/token/token.go @@ -51,6 +51,7 @@ const ( CASE WHEN IN + FOR RETRY THEN DO @@ -117,7 +118,7 @@ var typeNames = map[Type]string{ STRING: "STRING", STRBEG: "STRBEG", STRMID: "STRMID", STREND: "STREND", IDENT: "IDENT", CONST: "CONST", IVAR: "IVAR", CVAR: "CVAR", GVAR: "GVAR", SYMBOL: "SYMBOL", LABEL: "LABEL", REGEXP: "REGEXP", XSTRING: "XSTRING", WORDS: "WORDS", SYMBOLS: "SYMBOLS", DEF: "def", CLASS: "class", MODULE: "module", END: "end", IF: "if", ELSIF: "elsif", ELSE: "else", UNLESS: "unless", WHILE: "while", - UNTIL: "until", RETURN: "return", BREAK: "break", NEXT: "next", BEGIN: "begin", RESCUE: "rescue", ENSURE: "ensure", CASE: "case", WHEN: "when", IN: "in", RETRY: "retry", + UNTIL: "until", RETURN: "return", BREAK: "break", NEXT: "next", BEGIN: "begin", RESCUE: "rescue", ENSURE: "ensure", CASE: "case", WHEN: "when", IN: "in", FOR: "for", RETRY: "retry", THEN: "then", DO: "do", TRUE: "true", FALSE: "false", NIL: "nil", SELF: "self", SUPER: "super", YIELD: "yield", AND: "and", OR: "or", NOT: "not", ALIAS: "alias", UNDEF: "undef", @@ -143,7 +144,7 @@ var Keywords = map[string]Type{ "unless": UNLESS, "while": WHILE, "until": UNTIL, "return": RETURN, "break": BREAK, "next": NEXT, "begin": BEGIN, "rescue": RESCUE, "ensure": ENSURE, - "case": CASE, "when": WHEN, "in": IN, "retry": RETRY, + "case": CASE, "when": WHEN, "in": IN, "for": FOR, "retry": RETRY, "then": THEN, "do": DO, "true": TRUE, "false": FALSE, "nil": NIL, "self": SELF, "super": SUPER, "yield": YIELD, "and": AND, "or": OR, "not": NOT,