diff --git a/lambda_kwarg_numeric_singleton_test.go b/lambda_kwarg_numeric_singleton_test.go new file mode 100644 index 0000000..c940eb9 --- /dev/null +++ b/lambda_kwarg_numeric_singleton_test.go @@ -0,0 +1,732 @@ +package parser_test + +import ( + "math/big" + "strings" + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// This file covers eight refusals fixed together, each traced to one production +// in MRI's grammar. Every `accepted` entry below is a shape both ruby 4.0.5 +// parsers (prism and `--parser=parse.y`) accept, and every `refused` entry is one +// both of them refuse — checked shape by shape, not by exit status alone. + +// lambdaBlock digs the *ast.Block out of the `lambda { … }` call a stabby lambda +// desugars to. +func lambdaBlock(t *testing.T, src string) *ast.Block { + t.Helper() + call, ok := mustParseSingle(t, src).(*ast.Call) + if !ok { + t.Fatalf("Parse(%q): top is %T, want *ast.Call", src, mustParseSingle(t, src)) + } + if call.Name != "lambda" { + t.Fatalf("Parse(%q): call name = %q, want %q", src, call.Name, "lambda") + } + if call.Block == nil { + t.Fatalf("Parse(%q): call has no block", src) + } + return call.Block +} + +// --- 1. a keyword parameter first in a PAREN-LESS lambda parameter list --- + +// TestParenlessLambdaKeywordParams is the target defect: `-> x: 1 { x }` was +// refused with "expected {, got x (LABEL)". +// +// MRI's paren-less form is the plain `f_larglist: f_args` alternative (parse.y +// v3_4_0 5216) — the SAME `f_args` the parenthesised form takes — and +// `f_args: args_tail` (6473) makes a bare `args_tail` a complete parameter list. +// `args_tail` begins with `f_kwarg(f_kw)` (6371), `f_kw: f_label arg_value | +// f_label` (6611), `f_label: tLABEL` (6591). A tLABEL is therefore a legal FIRST +// token, exactly as `**kw` and `&b` already were. +// +// The parameter names carry the trailing-colon sentinel parseBlockParams uses for +// a keyword parameter, and Defaults is nil for a required one. +func TestParenlessLambdaKeywordParams(t *testing.T) { + cases := []struct { + src string + wantParams []string + wantDflt []bool // true where a default expression is present + }{ + {"-> x: { x }", []string{"x:"}, []bool{false}}, + {"-> x: 1 { x }", []string{"x:"}, []bool{true}}, + {"-> x: 1 do x end", []string{"x:"}, []bool{true}}, + {"-> a:, b: { }", []string{"a:", "b:"}, []bool{false, false}}, + {"-> x: 1, y: 2 { }", []string{"x:", "y:"}, []bool{true, true}}, + {"-> k: 1, **kw { }", []string{"k:", "**kw"}, []bool{true, false}}, + // A positional parameter first already worked; it must keep working, and it + // pins that the two entry paths agree on the recorded shape. + {"-> x, y: 2 { }", []string{"x", "y:"}, []bool{false, true}}, + // The parenthesised spelling is the other f_larglist alternative. + {"->(x: 1) { x }", []string{"x:"}, []bool{true}}, + } + for _, c := range cases { + blk := lambdaBlock(t, c.src) + if strings.Join(blk.Params, ",") != strings.Join(c.wantParams, ",") { + t.Errorf("Parse(%q): params = %v, want %v", c.src, blk.Params, c.wantParams) + continue + } + if len(blk.Defaults) != len(c.wantDflt) { + t.Errorf("Parse(%q): %d defaults, want %d", c.src, len(blk.Defaults), len(c.wantDflt)) + continue + } + for i, want := range c.wantDflt { + if got := blk.Defaults[i] != nil; got != want { + t.Errorf("Parse(%q): default %d present = %v, want %v", c.src, i, got, want) + } + } + } +} + +// TestParenlessLambdaKeywordParamBlockParam covers the `&b` tail after a keyword +// parameter (`args_tail: f_kwarg(f_kw) opt_f_block_arg`, 6376), which lands in +// BlockParam rather than Params. +func TestParenlessLambdaKeywordParamBlockParam(t *testing.T) { + blk := lambdaBlock(t, "-> k: 1, &b { }") + if strings.Join(blk.Params, ",") != "k:" { + t.Errorf("params = %v, want [k:]", blk.Params) + } + if blk.BlockParam != "b" { + t.Errorf("BlockParam = %q, want %q", blk.BlockParam, "b") + } +} + +// --- 2. negating an `r`/`i`-suffixed numeric literal --- + +// TestNegateRationalAndImaginaryLiteral covers the unchecked type assertion that +// ended negateLiteral: applyNumSuffix wraps the literal in a RationalLit and/or +// an ImaginaryLit, and the assertion was `n.(*ast.FloatLit)`. Every `-Nr`, `-Ni` +// and `-Nri` therefore panicked, surfacing as "internal parser error". +// +// The wrapper is preserved and the innermost numeric is negated. MRI sets a +// `minus` flag on the OUTERMOST node (`negate_lit`, parse.y v3_4_0: NODE_RATIONAL +// and NODE_IMAGINARY each carry their own), which for these shapes is the same +// number — (-3)/1 is (-3/1), and 0 + (-3/1)i is (0-(3/1)*i), the value ruby 4.0.5 +// prints for `p(-3ri)`. +func TestNegateRationalAndImaginaryLiteral(t *testing.T) { + t.Run("-3r is RationalLit(IntLit -3)", func(t *testing.T) { + r, ok := mustParseSingle(t, "-3r").(*ast.RationalLit) + if !ok { + t.Fatalf("top is %T, want *ast.RationalLit", mustParseSingle(t, "-3r")) + } + i, ok := r.Value.(*ast.IntLit) + if !ok || i.Value != -3 { + t.Fatalf("RationalLit.Value = %#v, want IntLit(-3)", r.Value) + } + }) + t.Run("-3i is ImaginaryLit(IntLit -3)", func(t *testing.T) { + im, ok := mustParseSingle(t, "-3i").(*ast.ImaginaryLit) + if !ok { + t.Fatalf("top is %T, want *ast.ImaginaryLit", mustParseSingle(t, "-3i")) + } + i, ok := im.Value.(*ast.IntLit) + if !ok || i.Value != -3 { + t.Fatalf("ImaginaryLit.Value = %#v, want IntLit(-3)", im.Value) + } + }) + t.Run("-3ri nests imaginary over rational", func(t *testing.T) { + im, ok := mustParseSingle(t, "-3ri").(*ast.ImaginaryLit) + if !ok { + t.Fatalf("top is %T, want *ast.ImaginaryLit", mustParseSingle(t, "-3ri")) + } + r, ok := im.Value.(*ast.RationalLit) + if !ok { + t.Fatalf("ImaginaryLit.Value = %T, want *ast.RationalLit", im.Value) + } + i, ok := r.Value.(*ast.IntLit) + if !ok || i.Value != -3 { + t.Fatalf("innermost = %#v, want IntLit(-3)", r.Value) + } + }) + t.Run("-0.5r negates the float inside", func(t *testing.T) { + r, ok := mustParseSingle(t, "-0.5r").(*ast.RationalLit) + if !ok { + t.Fatalf("top is %T, want *ast.RationalLit", mustParseSingle(t, "-0.5r")) + } + f, ok := r.Value.(*ast.FloatLit) + if !ok || f.Value != -0.5 { + t.Fatalf("RationalLit.Value = %#v, want FloatLit(-0.5)", r.Value) + } + }) + t.Run("a negated bignum with a suffix stays a bignum", func(t *testing.T) { + const src = "-99999999999999999999999r" + r, ok := mustParseSingle(t, src).(*ast.RationalLit) + if !ok { + t.Fatalf("top is %T, want *ast.RationalLit", mustParseSingle(t, src)) + } + bn, ok := r.Value.(*ast.BignumLit) + if !ok { + t.Fatalf("RationalLit.Value = %T, want *ast.BignumLit", r.Value) + } + want, _ := new(big.Int).SetString("-99999999999999999999999", 10) + if bn.Val.Cmp(want) != 0 { + t.Errorf("value = %s, want %s", bn.Val, want) + } + }) +} + +// --- 3. the `r`/`i` suffix on a radix-prefixed integer, and MRI's two suffix rules --- + +// TestNumericSuffixOnRadixLiteral covers `0x10r` and friends, which lexRadixInt +// simply did not read: parse.y v3_4_0 calls number_literal_suffix(NUM_SUFFIX_ALL) +// on the hex (9839), binary (9863), explicit-decimal (9892) and octal (9920) +// paths exactly as on the decimal one. +func TestNumericSuffixOnRadixLiteral(t *testing.T) { + cases := []struct { + src string + kind string // "r", "i", or "ri" + val int64 + }{ + {"0x10r", "r", 16}, + {"0b10i", "i", 2}, + {"0o7r", "r", 7}, + {"0d9i", "i", 9}, + {"0x10ri", "ri", 16}, + {"0xar", "r", 10}, // 'a' is a hex digit; 'r' is the suffix + {"0xai", "i", 10}, + } + for _, c := range cases { + n := mustParseSingle(t, c.src) + // Unwrap in the order applyNumSuffix wrapped: "r" innermost, "i" outermost. + if strings.Contains(c.kind, "i") { + im, ok := n.(*ast.ImaginaryLit) + if !ok { + t.Errorf("Parse(%q): top is %T, want *ast.ImaginaryLit", c.src, n) + continue + } + n = im.Value + } + if strings.Contains(c.kind, "r") { + r, ok := n.(*ast.RationalLit) + if !ok { + t.Errorf("Parse(%q): inner is %T, want *ast.RationalLit", c.src, n) + continue + } + n = r.Value + } + i, ok := n.(*ast.IntLit) + if !ok || i.Value != c.val { + t.Errorf("Parse(%q): innermost = %#v, want IntLit(%d)", c.src, n, c.val) + } + } +} + +// TestNumericSuffixRejectedByTrailingLetter pins the rule that makes +// `2rescue nil` a valid program: number_literal_suffix (parse.y v3_4_0 9044) +// rewinds and returns NO suffix when the character ending the scan is a letter, +// `_`, or non-ASCII. `2r` followed by `escue` would be a syntax error; `2` with a +// modifier `rescue` is what MRI parses, and `ruby -e 'p 2rescue nil'` prints 2. +func TestNumericSuffixRejectedByTrailingLetter(t *testing.T) { + // `p 2rescue nil` is `(p 2) rescue nil`: a Begin with one rescue clause. + b, ok := mustParseSingle(t, "p 2rescue nil").(*ast.Begin) + if !ok { + t.Fatalf("top is %T, want *ast.Begin (a modifier rescue)", mustParseSingle(t, "p 2rescue nil")) + } + if len(b.Rescues) != 1 { + t.Fatalf("%d rescue clauses, want 1", len(b.Rescues)) + } + call, ok := b.Body[0].(*ast.Call) + if !ok || call.Name != "p" || len(call.Args) != 1 { + t.Fatalf("Begin.Body[0] = %#v, want Call p with 1 arg", b.Body[0]) + } + // The argument must be the PLAIN integer 2 — not a rational. + if i, ok := call.Args[0].(*ast.IntLit); !ok || i.Value != 2 { + t.Fatalf("argument = %#v, want IntLit(2) with no suffix", call.Args[0]) + } +} + +// TestNumericSuffixIAfterRIsNotR pins the other half of that function: after an +// `i`, the `r` bit is masked off ("rational of complex is disallowed", 9053), so +// `2ri` carries both suffixes and `2ir` carries neither — and with `r` no longer +// acceptable, the trailing-letter rule then rejects the whole suffix, which is +// why ruby 4.0.5 refuses `p 2ir` under both its parsers. +func TestNumericSuffixIAfterRIsNotR(t *testing.T) { + // 2ri: rational inside imaginary. + im, ok := mustParseSingle(t, "2ri").(*ast.ImaginaryLit) + if !ok { + t.Fatalf("2ri: top is %T, want *ast.ImaginaryLit", mustParseSingle(t, "2ri")) + } + if _, ok := im.Value.(*ast.RationalLit); !ok { + t.Fatalf("2ri: ImaginaryLit.Value = %T, want *ast.RationalLit", im.Value) + } + // 2ir: refused, because the suffix scan rewinds and leaves the identifier `ir` + // sitting after the integer. + if _, err := parser.Parse("p 2ir"); err == nil { + t.Error("Parse(`p 2ir`): accepted, want a refusal (MRI refuses it too)") + } +} + +// --- 4. a singleton `def` whose receiver is separated by `::` --- + +// TestSingletonDefViaScopeOperator covers `def M::name(v)` and `def self::m`, +// which reach rbconfig.rb and mkmf.rb. MRI has one production for both +// separators: `defs_head: k_def singleton dot_or_colon def_name` (parse.y v3_4_0 +// 3405) with `dot_or_colon: '.' | tCOLON2` (6839). The `::` spelling must produce +// exactly the same MethodDef the `.` spelling does. +func TestSingletonDefViaScopeOperator(t *testing.T) { + t.Run("def self::m sets Singleton", func(t *testing.T) { + d, ok := mustParseSingle(t, "def self::m; end").(*ast.MethodDef) + if !ok { + t.Fatalf("top is %T, want *ast.MethodDef", mustParseSingle(t, "def self::m; end")) + } + if !d.Singleton || d.Recv != nil || d.Name != "m" { + t.Errorf("got Singleton=%v Recv=%#v Name=%q, want true/nil/\"m\"", d.Singleton, d.Recv, d.Name) + } + }) + t.Run("def M::name(v) sets a ConstRef receiver", func(t *testing.T) { + const src = "def M::name(v); end" + d, ok := mustParseSingle(t, src).(*ast.MethodDef) + if !ok { + t.Fatalf("top is %T, want *ast.MethodDef", mustParseSingle(t, src)) + } + c, ok := d.Recv.(*ast.ConstRef) + if !ok || c.Name != "M" { + t.Fatalf("Recv = %#v, want ConstRef{M}", d.Recv) + } + if d.Singleton { + t.Error("Singleton = true, want false (an explicit receiver, not self)") + } + if d.Name != "name" || len(d.Params) != 1 || d.Params[0] != "v" { + t.Errorf("got Name=%q Params=%v, want \"name\" [v]", d.Name, d.Params) + } + }) + t.Run(":: and . yield the same node", func(t *testing.T) { + dot, ok := mustParseSingle(t, "def M.name(v); end").(*ast.MethodDef) + if !ok { + t.Fatal("def M.name: not a MethodDef") + } + scope, ok := mustParseSingle(t, "def M::name(v); end").(*ast.MethodDef) + if !ok { + t.Fatal("def M::name: not a MethodDef") + } + if dot.Name != scope.Name || dot.Singleton != scope.Singleton || + len(dot.Params) != len(scope.Params) { + t.Errorf("`.` gave %#v, `::` gave %#v — the same production must agree", dot, scope) + } + }) + t.Run("keyword_variable receivers", func(t *testing.T) { + // `singleton: var_ref` and `var_ref: user_variable | keyword_variable` + // (parse.y v3_4_0 6711), so nil/true/false are receivers MRI parses. + for _, src := range []string{"def nil.m; end", "def true.m; end", "def false.m; end"} { + d, ok := mustParseSingle(t, src).(*ast.MethodDef) + if !ok { + t.Fatalf("Parse(%q): top is %T, want *ast.MethodDef", src, mustParseSingle(t, src)) + } + if d.Recv == nil { + t.Errorf("Parse(%q): Recv = nil, want a literal receiver node", src) + } + } + }) +} + +// --- 5. `when`/`in`/`case` with the value on the next line --- + +// TestWhenValueOnNextLine covers rubygems/request.rb:125. `case_body: k_when +// case_args then` (parse.y v3_4_0 6191) has no `terms` between `k_when` and +// `case_args`, so a newline there can only be legal because MRI's lexer discards +// it: `when` sets a value-expecting state and `parser_yylex`'s '\n' case does not +// emit a terminator in one. +func TestWhenValueOnNextLine(t *testing.T) { + const src = "case x\nwhen\n 1 then 2\nend" + c, ok := mustParseSingle(t, src).(*ast.Case) + if !ok { + t.Fatalf("top is %T, want *ast.Case", mustParseSingle(t, src)) + } + if len(c.Whens) != 1 { + t.Fatalf("%d when clauses, want 1", len(c.Whens)) + } + if len(c.Whens[0].Conds) != 1 { + t.Fatalf("%d conditions, want 1", len(c.Whens[0].Conds)) + } + if i, ok := c.Whens[0].Conds[0].(*ast.IntLit); !ok || i.Value != 1 { + t.Errorf("condition = %#v, want IntLit(1)", c.Whens[0].Conds[0]) + } + if len(c.Whens[0].Body) != 1 { + t.Errorf("%d body nodes, want 1 — the newline must join, not split", len(c.Whens[0].Body)) + } +} + +// TestCaseWhenWithoutTerminator covers `k_case terms?` (6136): the terminator +// after `case` is OPTIONAL, so a subject-less `case` may be followed by its first +// clause keyword with nothing in between. +func TestCaseWhenWithoutTerminator(t *testing.T) { + const src = "case when false then 1 end" + c, ok := mustParseSingle(t, src).(*ast.Case) + if !ok { + t.Fatalf("top is %T, want *ast.Case", mustParseSingle(t, src)) + } + if c.Subject != nil { + t.Errorf("Subject = %#v, want nil (the subject-less case)", c.Subject) + } + if len(c.Whens) != 1 { + t.Errorf("%d when clauses, want 1", len(c.Whens)) + } +} + +// TestClauseKeywordAsMethodNameDoesNotContinue is the regression this batch +// produced and then had to fix: once `in` continued a line, `def in` ⏎ `body` +// joined into `def in body` and prism/node.rb — 728 of 728 stdlib files +// otherwise — stopped parsing. A reserved word in a method-NAME position +// (MRI's EXPR_FNAME, after `def`, and EXPR_DOT, after a dot) is a name, not the +// operator it usually spells, so the line must not continue. +func TestClauseKeywordAsMethodNameDoesNotContinue(t *testing.T) { + for _, name := range []string{"in", "then", "case", "when", "if", "unless", "while", "until", "and", "or"} { + src := "class C\n def " + name + "\n other_loc.slice\n end\nend" + prog, err := parser.Parse(src) + if err != nil { + t.Errorf("Parse(def %s with a body on the next line): %v", name, err) + continue + } + cd, ok := prog.Body[0].(*ast.ClassDef) + if !ok { + t.Errorf("def %s: top is %T, want *ast.ClassDef", name, prog.Body[0]) + continue + } + d, ok := cd.Body[0].(*ast.MethodDef) + if !ok { + t.Errorf("def %s: class body[0] is %T, want *ast.MethodDef", name, cd.Body[0]) + continue + } + if d.Name != name { + t.Errorf("method name = %q, want %q", d.Name, name) + } + if len(d.Params) != 0 { + t.Errorf("def %s: %d params, want 0 — the next line is the BODY", name, len(d.Params)) + } + if len(d.Body) != 1 { + t.Errorf("def %s: %d body nodes, want 1", name, len(d.Body)) + } + } +} + +// --- 6. `**nil` in a block / lambda parameter list --- + +// TestBlockKeywordNoRest covers `-> (**nil) { }` and `{ |**nil| }`. MRI reaches +// it through `block_args_tail: f_any_kwrest opt_f_block_arg` (parse.y v3_4_0 +// 5006), `f_any_kwrest: f_kwrest | f_no_kwarg` and `f_no_kwarg: p_kwnorest: +// kwrest_mark keyword_nil` — the same marker a method list already accepted. +func TestBlockKeywordNoRest(t *testing.T) { + t.Run("stabby lambda", func(t *testing.T) { + blk := lambdaBlock(t, "-> (**nil) { }") + if strings.Join(blk.Params, ",") != "**nil" { + t.Errorf("params = %v, want [**nil]", blk.Params) + } + }) + t.Run("brace block with a positional first", func(t *testing.T) { + call, ok := mustParseSingle(t, "proc { |a, **nil| }").(*ast.Call) + if !ok || call.Block == nil { + t.Fatalf("top is %#v, want a call with a block", mustParseSingle(t, "proc { |a, **nil| }")) + } + if strings.Join(call.Block.Params, ",") != "a,**nil" { + t.Errorf("params = %v, want [a **nil]", call.Block.Params) + } + }) + t.Run("followed by a block param", func(t *testing.T) { + blk := lambdaBlock(t, "-> (a, **nil, &b) { }") + if strings.Join(blk.Params, ",") != "a,**nil" { + t.Errorf("params = %v, want [a **nil]", blk.Params) + } + if blk.BlockParam != "b" { + t.Errorf("BlockParam = %q, want %q", blk.BlockParam, "b") + } + }) + t.Run("an anonymous ** is still distinct from **nil", func(t *testing.T) { + blk := lambdaBlock(t, "-> (**) { }") + if strings.Join(blk.Params, ",") != "**" { + t.Errorf("params = %v, want [**]", blk.Params) + } + }) +} + +// --- 7. BEGIN { } and END { } --- + +// TestBeginBlockHoistsToProgramFront covers `BEGIN { }`: MRI accumulates the +// block into `p->eval_tree_begin` and runs it before the program body, leaving a +// void `NEW_BEGIN(0)` where it stood (parse.y v3_4_0 3020, 3029). Both halves are +// reproduced, so the statement that textually FOLLOWS a BEGIN runs after it. +func TestBeginBlockHoistsToProgramFront(t *testing.T) { + const src = "p 2\nBEGIN { p 1 }" + prog := mustParse(t, src) + if len(prog.Body) != 3 { + t.Fatalf("%d top-level nodes, want 3 (hoisted body, `p 2`, the void Begin)", len(prog.Body)) + } + first, ok := prog.Body[0].(*ast.Call) + if !ok || first.Name != "p" { + t.Fatalf("Body[0] = %#v, want the hoisted `p 1`", prog.Body[0]) + } + if i, ok := first.Args[0].(*ast.IntLit); !ok || i.Value != 1 { + t.Errorf("Body[0] argument = %#v, want IntLit(1): the BEGIN body must come FIRST", first.Args[0]) + } + second, ok := prog.Body[1].(*ast.Call) + if !ok { + t.Fatalf("Body[1] = %#v, want `p 2`", prog.Body[1]) + } + if i, ok := second.Args[0].(*ast.IntLit); !ok || i.Value != 2 { + t.Errorf("Body[1] argument = %#v, want IntLit(2)", second.Args[0]) + } + // The placeholder left behind evaluates to nil and must carry no body. + last, ok := prog.Body[2].(*ast.Begin) + if !ok { + t.Fatalf("Body[2] = %#v, want an empty *ast.Begin placeholder", prog.Body[2]) + } + if len(last.Body) != 0 || len(last.Rescues) != 0 { + t.Errorf("placeholder = %#v, want an empty Begin (MRI's NEW_BEGIN(0))", last) + } +} + +// TestEndBlockIsAtExit covers `END { }`, desugared to the replacement MRI's own +// warning names (`rb_warn0("END in method; use at_exit")`, parse.y v3_4_0 3196). +func TestEndBlockIsAtExit(t *testing.T) { + call, ok := mustParseSingle(t, "END { p 1 }").(*ast.Call) + if !ok { + t.Fatalf("top is %T, want *ast.Call", mustParseSingle(t, "END { p 1 }")) + } + if call.Name != "at_exit" || call.Recv != nil { + t.Fatalf("got %q with Recv %#v, want a receiver-less at_exit", call.Name, call.Recv) + } + if call.Block == nil || len(call.Block.Body) != 1 { + t.Fatalf("block = %#v, want one body statement", call.Block) + } + if len(call.Args) != 0 { + t.Errorf("%d arguments, want 0", len(call.Args)) + } +} + +// TestBeginBlockNestingRule covers where BEGIN is and is not allowed. Every +// nested body reaches `stmt_or_begin` (parse.y v3_4_0 3093), whose keyword_BEGIN +// alternative exists only to emit "BEGIN is permitted only at toplevel" — but +// `begin_block` itself reads a `top_compstmt` (3029), so a BEGIN inside a BEGIN +// is at top level again and parses. +func TestBeginBlockNestingRule(t *testing.T) { + if _, err := parser.Parse("BEGIN { BEGIN { } }"); err != nil { + t.Errorf("Parse(`BEGIN { BEGIN { } }`): %v — begin_block reads a top_compstmt", err) + } + for _, src := range []string{ + "def f; BEGIN { }; end", + "if true; BEGIN { }; end", + "1.times { BEGIN { } }", + "class C; BEGIN{}; end", + "(BEGIN { })", + } { + _, err := parser.Parse(src) + if err == nil { + t.Errorf("Parse(%q): accepted, want refused (MRI: BEGIN is permitted only at toplevel)", src) + continue + } + if !strings.Contains(err.Error(), "toplevel") { + t.Errorf("Parse(%q): %v — want the message to name the toplevel rule", src, err) + } + } + // END is an ordinary `stmt` (3193), legal in a method body; MRI only warns. + if _, err := parser.Parse("def f; END { }; end"); err != nil { + t.Errorf("Parse(`def f; END { }; end`): %v — END is a plain stmt", err) + } +} + +// --- 8. an operator method name after a dot --- + +// TestOperatorMethodNameAfterDot covers `6543.21.%(137)` from +// library/bigdecimal/shared/modulo.rb and `2./(3)`. The four %-literal openers +// and the regexp opener were each guarded by `prevType != token.DEF`, which named +// `def` but not a dot, so a `%` after `.` opened a %Q string and a `/` opened a +// regexp. MRI's states for a method-name position are EXPR_FNAME and EXPR_DOT, +// and neither belongs to the EXPR_BEG_ANY set `parser_yylex` tests before calling +// parse_percent or opening a regexp. +func TestOperatorMethodNameAfterDot(t *testing.T) { + cases := []struct{ src, name string }{ + {"6543.21.%(137)", "%"}, + {"2.%(3)", "%"}, + {"2./(3)", "/"}, + {"x.%(3)", "%"}, + {"x&.%(3)", "%"}, + {"x./(3)", "/"}, + } + for _, c := range cases { + call, ok := mustParseSingle(t, c.src).(*ast.Call) + if !ok { + t.Errorf("Parse(%q): top is %T, want *ast.Call", c.src, mustParseSingle(t, c.src)) + continue + } + if call.Name != c.name { + t.Errorf("Parse(%q): method = %q, want %q", c.src, call.Name, c.name) + } + if call.Recv == nil { + t.Errorf("Parse(%q): Recv = nil, want the literal/variable before the dot", c.src) + } + if len(call.Args) != 1 { + t.Errorf("Parse(%q): %d args, want 1", c.src, len(call.Args)) + } + } +} + +// TestPercentAndRegexpStillOpenInValuePosition is the other side of that +// predicate: where a VALUE is expected, `%(` and `/` must still open literals. +// Without this the fix above could be satisfied by never opening them at all. +func TestPercentAndRegexpStillOpenInValuePosition(t *testing.T) { + if s, ok := mustParseSingle(t, "%(hi)").(*ast.StringLit); !ok || s.Value != "hi" { + t.Errorf("Parse(`%%(hi)`) = %#v, want StringLit(\"hi\")", mustParseSingle(t, "%(hi)")) + } + if _, ok := mustParseSingle(t, "/a/").(*ast.RegexpLit); !ok { + t.Errorf("Parse(`/a/`) = %T, want *ast.RegexpLit", mustParseSingle(t, "/a/")) + } + if _, ok := mustParseSingle(t, "%w[a b]").(*ast.ArrayLit); !ok { + t.Errorf("Parse(`%%w[a b]`) = %T, want *ast.ArrayLit", mustParseSingle(t, "%w[a b]")) + } + // `def %(o)` and `def /(o)` name the operator methods, unchanged. + for _, src := range []string{"def %(o); end", "def /(o); end"} { + d, ok := mustParseSingle(t, src).(*ast.MethodDef) + if !ok { + t.Errorf("Parse(%q): top is %T, want *ast.MethodDef", src, mustParseSingle(t, src)) + continue + } + if len(d.Params) != 1 { + t.Errorf("Parse(%q): %d params, want 1", src, len(d.Params)) + } + } +} + +// --- 9. "can't define singleton method for literals" --- + +// TestSingletonDefLiteralReceiverRefused covers MRI's own check on the `singleton` +// production, over a fixed node list (parse.y v3_4_0 6724-6744: NODE_STR, DSTR, +// XSTR, DXSTR, REGX, DREGX, SYM, DSYM, LINE, FILE, ENCODING, INTEGER, FLOAT, +// RATIONAL, IMAGINARY, LIST, ZLIST). +// +// Ruby 4.0 MOVED this check: v3_4_0 applies it only to the `'(' expr rparen` +// alternative, while 4.0.5 restructured `singleton` into +// `value_expr(singleton_expr)` and applies it to the bare `var_ref` spelling too. +// `def __FILE__.m` is the shape that distinguishes them — it parses under the +// v3_4_0 grammar and is refused by ruby 4.0.5 under BOTH its parsers, which is +// the oracle this repo measures against. +func TestSingletonDefLiteralReceiverRefused(t *testing.T) { + for _, src := range []string{ + "def __FILE__.m; end", "def __LINE__.m; end", "def __ENCODING__.m; end", + "def (1).m; end", "def (1.5).m; end", "def (2r).m; end", "def (2i).m; end", + `def ("s").m; end`, `def ("a#{1}").m; end`, "def (:sym).m; end", + "def ([]).m; end", "def ([1]).m; end", "def (/re/).m; end", "def (`x`).m; end", + } { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q): accepted, want refused (MRI: can't define singleton method for literals)", src) + } + } + // Not in MRI's list: nil/true/false/self and a hash, plus any real expression. + for _, src := range []string{ + "def (nil).m; end", "def (self).m; end", "def ({}).m; end", + "def (obj).m; end", "def (foo.bar).m; end", + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v — MRI accepts this receiver", src, err) + } + } +} + +// TestSingletonDefReceiverIsNotAPath pins an over-acceptance this batch removed: +// MRI's receiver is a single `var_ref`, never a constant path, so +// `def M::Bar::baz` is a SyntaxError ("unexpected ::, expecting '\n' or ';'"). +// It used to be accepted here by mis-reading it as `def M` with a `::Bar::baz` +// body — a definition of the wrong method, silently. +func TestSingletonDefReceiverIsNotAPath(t *testing.T) { + for _, src := range []string{"def M::Bar::baz; end", "def obj&.m; end"} { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q): accepted, want refused (MRI refuses it under both parsers)", src) + } + } +} + +// --- the standing accept/refuse tables, extended --- + +// TestWt47AcceptTable and TestWt47RefuseTable extend the repo's existing +// accept/refuse tables with this batch's shapes plus the controls that must not +// move. Each entry was judged by BOTH ruby 4.0.5 front ends (prism and +// `--parser=parse.y`) and they agree on every one; shapes where they disagree +// (`a[foo bar do end]`, `f(foo bar do end)` and the `not`-in-`expr` family) are +// deliberately absent, because there is nothing there to assert. +func TestWt47AcceptTable(t *testing.T) { + accepted := []string{ + // 1. paren-less lambda keyword parameters + "-> x: { x }", "-> x: 1 { x }", "-> x: 1 do x end", "-> a:, b: { }", + "-> x: 1, y: 2 { }", "-> k: 1, **kw { }", "-> k: 1, &b { }", + "-> x, y: 2 { }", "-> x: 1 { x }.call(x: 2)", "->(x: 1) { x }", + // 2. negated suffixed numerics + "p(-3r)", "p(-3i)", "p(-3ri)", "p(-0.5r)", "p(-2.5i)", "p(-3r ** 2)", "p(-3r.to_s)", + // 3. radix literals with a suffix + "p 0x10r", "p 0b10i", "p 0o7r", "p 0d9i", "p 0x10ri", "p 0xar", + "p(-0x10r)", "p 1_000r", "p 2rescue nil", + // 4. `::` singleton def + "def M::name(v); end", "def M::Name(v); end", "def self::m; end", + "def obj::m; end", "def @iv::m; end", "def $g::m; end", "def @@cv::m; end", + "def (foo)::m; end", "def M::m=(v); end", "def M::[](i); end", + "def M::+(o); end", "def M::class; end", "def M::%(o); end", + "def nil.m; end", "def true.m; end", "def false.m; end", + // 5. a clause value on the next line + "case x\nwhen\n 1\n 2\nend", + "case x\nwhen\n 1 then 2\nend", + "case x\nwhen\n 1, 2\n 3\nend", + "case\n x\nwhen 1\n 2\nend", + "case h\nin\n {}\n 1\nend", + "for i in\n [1]\n p i\nend", + "case when false then 1 end", + "case\nwhen false then 1\nend", + "if x\nthen\n 1\nend", + "case x\nwhen 1\nthen 2\nend", + "class C\n def in\n in_loc.slice\n end\nend", + "class C\n def then\n then_loc.slice\n end\nend", + // 6. **nil + "-> (**nil) { }", "lambda { |**nil| }", "proc { |a, **nil| }", + "-> (a, **nil, &b) { }", "def f(**nil); end", + // 7. BEGIN / END + "BEGIN { }", "END { }", "BEGIN { p 1 }\nEND { p 2 }", "BEGIN { BEGIN { } }", + "p 2\nBEGIN { p 1 }", "def f; END { }; end", + // 8. an operator method name after a dot, and the value-position literals + "p 6543.21.%(137)", "p 2.%(3)", "p 2./(3)", "p x.%(3)", "p x&.%(3)", + "p %(hi)", `p(/a/ =~ "a")`, "p %w[a b]", + "class C; def %(o); end; end", "class C; def /(o); end; end", + // controls from the standing table that must stay accepted + "a[foo bar]", "f(foo bar, 1)", "x = (y do end)", "[defined? foo]", + "{k: defined? foo}", "p((x rescue y))", + // shapes that fail for SEMANTIC reasons unless written inside the context + // that makes them legal — the distinction `ruby -c` alone cannot make + "def f; yield 1; end", "while true; break; end", "[1].each { next }", + "def f; return 1; end", "class C; def f; super; end; end", + } + for _, src := range accepted { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v — MRI accepts this under both parsers", src, err) + } + } +} + +func TestWt47RefuseTable(t *testing.T) { + refused := []string{ + // over-acceptances removed by this batch + "def M::Bar::baz; end", "def obj&.m; end", + // dot_or_colon is the ONLY thing that may follow a parenthesised receiver + "def (foo)m; end", "def (foo) m; end", + "def __FILE__.m; end", "def __LINE__.m; end", "def __ENCODING__.m; end", + "def (1).m; end", "def (1.5).m; end", `def ("s").m; end`, `def ("a#{1}").m; end`, + "def (:sym).m; end", "def ([]).m; end", "def ([1]).m; end", "def (/re/).m; end", + "def (2r).m; end", "def (2i).m; end", "def (`x`).m; end", + // BEGIN outside a top_stmts list, and the `do` spelling of either block + "def f; BEGIN { }; end", "if true; BEGIN { }; end", "1.times { BEGIN { } }", + "class C; BEGIN{}; end", "(BEGIN { })", "BEGIN do end", "END do end", + // `r` after `i` is not a suffix pair, and the whole suffix is then rejected + "p 2ir", + // a command call in an `arg` position, even where `yield` itself is legal: + // `[yield 1]` is the `[foo bar]` rule, not a semantic refusal + "def f; [yield 1]; end", + // `{k: not x}` — both MRI parsers refuse this one, unlike the rest of the + // `not`-in-`expr` family, where they disagree and nothing is asserted + "{k: not x}", + // controls from the standing table that must stay refused + "while begin; y do end; end; end", "while y do end; end", "[foo bar]", + "if x end", "while x end", "case x; when 1 end", "p(x rescue y)", + } + for _, src := range refused { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q): accepted, want refused (MRI refuses it under both parsers)", src) + } + } +} diff --git a/lexer/lexer.go b/lexer/lexer.go index 7aadae6..b9a0fa4 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -50,6 +50,13 @@ type Lexer struct { // trailing one continue the line; in operand position those open a block-param // list, a block-pass, a heredoc, or a pattern pin instead. prevBinary bool + // prevFname records whether the last token stood in a method-NAME position — + // after `def`, or after `.`/`&.`/`::` (see atMethodName). A reserved word there + // is a method name, not the operator it usually spells, so a line ending in it + // does not continue: `def in`⏎`in_loc.slice` and `def then`⏎`…` (both in + // prism/node.rb) are a `def` whose body starts on the next line, where a + // statement-position `in`⏎`PATTERN` is one clause spanning two lines. + prevFname bool // pendingBinary is set within lexToken when the token being emitted is an // ambiguous operator in binary position; next() latches it into prevBinary. pendingBinary bool @@ -127,7 +134,14 @@ func (l *Lexer) Tokenize() []token.Token { func (l *Lexer) next() token.Token { l.pendingBinary = false l.pendingSetterAssign = false + // Whether the token about to be lexed stands in a method-NAME position has to + // be captured before lexToken moves prevType on, because isContinuationOp is + // asked about it one token later: a keyword used as a method name + // (`def in`, `x.then`) is not the keyword operator and does not continue the + // line, where the same spelling in statement position does. + fname := l.atMethodName() t := l.lexToken() + l.prevFname = fname l.prevType = t.Type l.prevBinary = l.pendingBinary l.setterAssign = l.pendingSetterAssign @@ -155,6 +169,26 @@ func (l *Lexer) regexAmbiguousAfterValueKeyword() bool { return l.prevType == token.SUPER || l.prevType == token.YIELD } +// atMethodName reports whether the cursor sits where a method NAME is expected, +// so an operator character there spells that name instead of opening a literal. +// MRI's two lexer states for this are EXPR_FNAME (just after `def`) and EXPR_DOT +// (just after `.`, `&.` or `::`), and neither belongs to EXPR_BEG_ANY — the set +// `parser_yylex` tests before it will call `parse_percent` for a `%` or open a +// regexp for a `/`. So `def %(o)` and `6543.21.%(137)` both name the modulo +// method, and `def /(o)` and `6.0./(2)` both name division, where a value +// position would have read `%(…)` as a %Q string and `/…/` as a regexp. +// +// One predicate rather than a repeated `prevType != token.DEF` at each call site: +// the previous form named only `def`, so all four literal openers silently +// carried the wrong rule after a dot, and `x.%(1)` / `x./(1)` were refused. +func (l *Lexer) atMethodName() bool { + switch l.prevType { + case token.DEF, token.DOT, token.SAFEDOT, token.SCOPE: + return true + } + return false +} + // isContinuationOp reports whether a line ending in token type t is incomplete, // so the trailing newline continues onto the next line rather than terminating // the statement. These are infix operators (plus comma and a trailing dot) that @@ -172,7 +206,18 @@ func isContinuationOp(t token.Type) bool { // Trailing low-precedence keyword operators and modifiers whose operand / // condition may sit on the next line: `... or\n fail`, `... and\n x`, // `do_it unless\n cond`, `foo if\n bar` (MRI joins these lines too). - token.AND, token.OR, token.IF, token.UNLESS, token.WHILE, token.UNTIL: + token.AND, token.OR, token.IF, token.UNLESS, token.WHILE, token.UNTIL, + // The clause keywords whose value also may sit on the next line. MRI does + // not list these anywhere: each sets a value-expecting lexer state, and + // `parser_yylex`'s '\n' case discards a newline in such a state, so the + // terminator the grammar wants is simply never emitted. `case_body: k_when + // case_args then` (parse.y v3_4_0 6191) has no `terms` between `k_when` and + // `case_args`, so `when`⏎`VALUE then …` can only parse if the newline is + // swallowed — which is the shape rubygems/request.rb:125 uses. `in` covers + // both `for x in`⏎`list` and a pattern clause `in`⏎`PATTERN`; `case` covers + // `case`⏎`subject` (and `k_case terms?` at 6136 already allows the + // subject-less `case`⏎`when …`). + token.WHEN, token.IN, token.CASE: return true } return false @@ -195,25 +240,26 @@ func (l *Lexer) lexToken() token.Token { switch { case c == 0: return mk(token.EOF, "") - case c == '/' && l.state == exprBegin && l.prevType != token.DEF && + case c == '/' && l.state == exprBegin && !l.atMethodName() && !(l.regexAmbiguousAfterValueKeyword() && spaceBefore && l.peek2() == ' '): // At expression-begin position a '/' opens a regexp literal, not division - // (the same disambiguation MRI uses via its lexer state). The one exception - // is right after `def`, where `/` names the division-operator method - // (`def /(other)`) rather than opening a pattern. + // (the same disambiguation MRI uses via its lexer state). The exception is a + // method-name position — after `def` or after a `.`/`&.` — where `/` names + // the division-operator method (`def /(other)`, `6.0./(2)`) rather than + // opening a pattern. // // `super`/`yield` may be used as bare values (`super / x`, `yield / 2`), // where a `/` spaced on both sides is division, not a regexp argument. The // `spaceBefore && spaceAfter` heuristic keeps `super /re/` (arg, no space // after) a regexp while making `super / x` division — matching MRI. return l.lexRegexp(spaceBefore, line, col) - case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): + case c == '%' && !l.atMethodName() && l.percentBeginsLiteral(spaceBefore) && l.atPercentArray(): // %w[…] / %i[…] / %W[…] / %I[…] word- and symbol-array literals. return l.lexPercentArray(spaceBefore, line, col) - case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentRXS(): + case c == '%' && !l.atMethodName() && l.percentBeginsLiteral(spaceBefore) && l.atPercentRXS(): // %r{…}flags regexp, %x{…} backtick command, %s{…} symbol literals. return l.lexPercentRXS(spaceBefore, line, col) - case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): + case c == '%' && !l.atMethodName() && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): // %q(…) / %Q(…) / %(…) / %W(…) string literals. return l.lexPercentString(spaceBefore, line, col) case c == '?' && l.charLiteralBegins(spaceBefore) && l.atCharLiteral(): @@ -249,7 +295,7 @@ func (l *Lexer) lexToken() token.Token { // Trailing-operator continuation: a line ending in an infix operator // (`a ||`, `x +`, a trailing comma, …) is incomplete and joins the next // line. `;` is an explicit terminator and is never suppressed this way. - if c == '\n' && isContinuationOp(l.prevType) && !l.setterAssign { + if c == '\n' && isContinuationOp(l.prevType) && !l.setterAssign && !l.prevFname { return l.next() } // The ambiguous bitwise/shift operators `|`/`&`/`^`/`<<` continue a line @@ -711,17 +757,7 @@ func (l *Lexer) lexNumber(spaceBefore bool, line, col int) token.Token { } } lit := stripUnderscores(string(l.src[start:l.pos])) - // Trailing `r` (rational) and/or `i` (imaginary) suffixes: `2r`, `0.5r`, - // `3i`, `2.5ri`. Recorded in Flags so the parser wraps the literal accordingly. - suffix := "" - if l.peek() == 'r' { - l.advance() - suffix += "r" - } - if l.peek() == 'i' { - l.advance() - suffix += "i" - } + suffix := l.numberLiteralSuffix() l.state = exprEnd tt := token.INT if isFloat { @@ -730,6 +766,49 @@ func (l *Lexer) lexNumber(spaceBefore bool, line, col int) token.Token { return token.Token{Type: tt, Lit: lit, Flags: suffix, Line: line, Col: col, SpaceBefore: spaceBefore} } +// numberLiteralSuffix consumes a trailing `r` (rational) and/or `i` (imaginary) +// suffix at the cursor and returns it ("", "r", "i" or "ri"), mirroring MRI's +// number_literal_suffix (parse.y v3_4_0 9044-9071). Two details of that function +// are load-bearing and are the reason this is one helper rather than an inline +// test at each numeric path: +// +// - an `r` after an `i` is masked off ("rational of complex is disallowed"), so +// `2ri` carries both suffixes while `2ir` carries neither; +// - if the character that ends the scan is a letter, `_`, or non-ASCII, the +// WHOLE suffix is rejected and the cursor rewinds (`p->lex.pcur = lastp`). +// That is why `2rescue nil` is `2` followed by the modifier `rescue` and not +// the rational `2r` followed by `escue`, and why `2x` is `2` then `x`. +// +// Every numeric path (decimal, and each radix prefix — parse.y calls it at 9839, +// 9863, 9892 and 9920) uses NUM_SUFFIX_ALL, so no caller needs a mask argument. +func (l *Lexer) numberLiteralSuffix() string { + startPos, startCol := l.pos, l.col + suffix := "" + mayI, mayR := true, true + for { + switch c := l.peek(); { + case c == 'i' && mayI: + mayI, mayR = false, false // `2ri` is legal; `2ir` is not + suffix += "i" + l.advance() + continue + case c == 'r' && mayR: + mayR = false + suffix += "r" + l.advance() + continue + case isIdentStart(c): // letters, `_`, and every non-ASCII byte + // The suffix scan ran into a letter or `_`: reject the whole suffix and + // rewind, so the digits stand alone and the letters lex as their own + // token. No suffix character is ever a newline, so the column rewinds by + // the same amount as the position. + l.pos, l.col = startPos, startCol + return "" + } + return suffix + } +} + // lexRadixInt lexes a prefixed integer literal (cursor on the leading '0'). The // emitted Lit keeps a Go-recognisable lowercase prefix (0x/0o/0b) so the parser // can decode it with base 0; an explicit-decimal 0d literal drops its prefix. @@ -751,15 +830,23 @@ func (l *Lexer) lexRadixInt(spaceBefore bool, line, col int) token.Token { } digits = append(digits, l.advance()) } - l.state = exprEnd if len(digits) == 0 { + l.state = exprEnd return token.Token{Type: token.ILLEGAL, Lit: "invalid numeric literal", Line: line, Col: col, SpaceBefore: spaceBefore} } + // A radix-prefixed integer takes the same `r`/`i` suffixes as a decimal one: + // parse.y v3_4_0 calls number_literal_suffix(NUM_SUFFIX_ALL) on the hex (9839), + // binary (9863), explicit-decimal (9892) and octal (9920) paths exactly as it + // does on the decimal one, so `0x10r` is (16/1) and `0b10i` is (0+2i). The + // suffix letters cannot be mistaken for digits: `r` and `i` are outside every + // radix's digit set, so the digit scan above always stops before them. + suffix := l.numberLiteralSuffix() + l.state = exprEnd lit := string(digits) if kind != 'd' { lit = "0" + string(kind) + lit } - return token.Token{Type: token.INT, Lit: lit, Line: line, Col: col, SpaceBefore: spaceBefore} + return token.Token{Type: token.INT, Lit: lit, Flags: suffix, Line: line, Col: col, SpaceBefore: spaceBefore} } // radixDigit returns the digit-membership test for a radix prefix letter. diff --git a/lexer/method_name_and_num_suffix_test.go b/lexer/method_name_and_num_suffix_test.go new file mode 100644 index 0000000..b98b046 --- /dev/null +++ b/lexer/method_name_and_num_suffix_test.go @@ -0,0 +1,168 @@ +package lexer + +import ( + "testing" + + "github.com/go-ruby-parser/parser/token" +) + +// types returns the token types of src, without the trailing EOF. +func types(src string) []token.Type { + toks := New(src).Tokenize() + out := make([]token.Type, 0, len(toks)) + for _, t := range toks { + if t.Type == token.EOF { + break + } + out = append(out, t.Type) + } + return out +} + +// TestOperatorAfterDotIsAMethodName covers atMethodName: in a method-NAME +// position — MRI's EXPR_FNAME after `def`, EXPR_DOT after `.`, `&.` or `::` — +// a `%` or `/` spells the method's name, so neither may open a literal. Neither +// state belongs to EXPR_BEG_ANY, the set parser_yylex tests before it calls +// parse_percent or opens a regexp. +// +// Each case is a three-token prefix: the receiver/keyword, the separator, and the +// operator that must arrive as an OPERATOR token rather than a STRING or REGEXP. +func TestOperatorAfterDotIsAMethodName(t *testing.T) { + cases := []struct { + src string + at int // index of the token that must be the operator, not a literal + want token.Type + }{ + {"x.%(1)", 2, token.PERCENT}, + {"x&.%(1)", 2, token.PERCENT}, + {"X::%(1)", 2, token.PERCENT}, + {"def %(o)", 1, token.PERCENT}, + {"x./(1)", 2, token.SLASH}, + {"x&./(1)", 2, token.SLASH}, + {"X::/(1)", 2, token.SLASH}, + {"def /(o)", 1, token.SLASH}, + } + for _, c := range cases { + got := types(c.src) + if len(got) <= c.at { + t.Errorf("%q: %d tokens, want more than %d", c.src, len(got), c.at) + continue + } + if got[c.at] != c.want { + t.Errorf("%q: token %d = %s, want %s (a literal opened instead of a method name)", + c.src, c.at, got[c.at], c.want) + } + } +} + +// TestPercentAndSlashStillOpenLiteralsElsewhere is the other side of that +// predicate: outside a method-name position the same characters must still open +// their literals, so the fix above cannot be satisfied by never opening them. +func TestPercentAndSlashStillOpenLiteralsElsewhere(t *testing.T) { + if got := types("%(hi)"); len(got) != 1 || got[0] != token.STRING { + t.Errorf("%%(hi): tokens = %v, want one STRING", got) + } + if got := types("/re/"); len(got) != 1 || got[0] != token.REGEXP { + t.Errorf("/re/: tokens = %v, want one REGEXP", got) + } + if got := types("a % b"); len(got) != 3 || got[1] != token.PERCENT { + t.Errorf("a %% b: tokens = %v, want the modulo operator in the middle", got) + } +} + +// TestNumberLiteralSuffix covers numberLiteralSuffix, which mirrors MRI's +// number_literal_suffix (parse.y v3_4_0 9044-9071). Three behaviours are pinned, +// and the last two are the ones a simpler "consume r then consume i" reading gets +// wrong: +// +// - `r`, `i` and `ri` are the accepted suffixes, on decimal AND radix literals; +// - an `r` after an `i` is masked off, so `2ir` is not a suffix pair; +// - a letter, `_` or non-ASCII byte ending the scan rejects the WHOLE suffix and +// rewinds, which is why `2rescue nil` is the integer 2 with a modifier rescue. +func TestNumberLiteralSuffix(t *testing.T) { + cases := []struct { + src string + wantLit string + wantFlags string + wantNext token.Type // the token after the number (EOF-trimmed away if absent) + wantNextIs bool + }{ + {src: "2r", wantLit: "2", wantFlags: "r"}, + {src: "2i", wantLit: "2", wantFlags: "i"}, + {src: "2ri", wantLit: "2", wantFlags: "ri"}, + {src: "0x10r", wantLit: "0x10", wantFlags: "r"}, + {src: "0b10i", wantLit: "0b10", wantFlags: "i"}, + {src: "0o7r", wantLit: "0o7", wantFlags: "r"}, + {src: "0d9i", wantLit: "9", wantFlags: "i"}, + {src: "0x10ri", wantLit: "0x10", wantFlags: "ri"}, + // `i` first: the `r` bit is cleared, so the trailing `r` is a letter that + // rejects the whole suffix and rewinds — `2ir` is `2` then the name `ir`. + {src: "2ir", wantLit: "2", wantFlags: "", wantNext: token.IDENT, wantNextIs: true}, + // The rewind that makes `2rescue nil` legal. + {src: "2rescue", wantLit: "2", wantFlags: "", wantNext: token.RESCUE, wantNextIs: true}, + // A letter with no suffix character at all takes the same path. + {src: "2x", wantLit: "2", wantFlags: "", wantNext: token.IDENT, wantNextIs: true}, + // A `_` is in MRI's reject set too. + {src: "2r_x", wantLit: "2", wantFlags: "", wantNext: token.IDENT, wantNextIs: true}, + // A non-letter after the suffix simply ends the scan. + {src: "2r.to_s", wantLit: "2", wantFlags: "r", wantNext: token.DOT, wantNextIs: true}, + {src: "2 + 1", wantLit: "2", wantFlags: "", wantNext: token.PLUS, wantNextIs: true}, + } + for _, c := range cases { + toks := New(c.src).Tokenize() + if len(toks) == 0 { + t.Errorf("%q: no tokens", c.src) + continue + } + if toks[0].Type != token.INT { + t.Errorf("%q: first token = %s, want INT", c.src, toks[0].Type) + continue + } + if toks[0].Lit != c.wantLit { + t.Errorf("%q: Lit = %q, want %q", c.src, toks[0].Lit, c.wantLit) + } + if toks[0].Flags != c.wantFlags { + t.Errorf("%q: Flags = %q, want %q", c.src, toks[0].Flags, c.wantFlags) + } + if c.wantNextIs { + if len(toks) < 2 { + t.Errorf("%q: only %d tokens, want a following %s", c.src, len(toks), c.wantNext) + continue + } + if toks[1].Type != c.wantNext { + t.Errorf("%q: second token = %s, want %s", c.src, toks[1].Type, c.wantNext) + } + } + } +} + +// TestClauseKeywordInMethodNamePositionDoesNotContinueTheLine covers prevFname: +// `when`, `in` and `case` continue a line the way `if` and `while` already did, +// but a reserved word in a method-NAME position is a NAME and must not. Without +// this, `def in` followed by a body on the next line joined into one line and +// prism/node.rb stopped parsing. +func TestClauseKeywordInMethodNamePositionDoesNotContinueTheLine(t *testing.T) { + for _, name := range []string{"in", "when", "case", "then", "if", "while", "and"} { + got := types("def " + name + "\n1\n") + // def, NAME, NEWLINE, 1, NEWLINE — the newline after the name must survive. + if len(got) < 3 { + t.Errorf("def %s: tokens = %v, want at least 3", name, got) + continue + } + if got[2] != token.NEWLINE { + t.Errorf("def %s: third token = %s, want NEWLINE (the line must not continue)", name, got[2]) + } + } + // In statement position the same keywords DO continue the line, so the guard is + // not simply switching the continuation off. + for _, src := range []string{"case x\nwhen\n1\n", "for i in\n[1]\n"} { + got := types(src) + for i, tt := range got { + if tt == token.WHEN || tt == token.IN { + if i+1 < len(got) && got[i+1] == token.NEWLINE { + t.Errorf("%q: a NEWLINE survived directly after %s, want it joined", src, tt) + } + } + } + } +} diff --git a/negate_literal_guard_internal_test.go b/negate_literal_guard_internal_test.go new file mode 100644 index 0000000..7ab420a --- /dev/null +++ b/negate_literal_guard_internal_test.go @@ -0,0 +1,45 @@ +package parser + +import ( + "strings" + "testing" + + "github.com/go-ruby-parser/parser/ast" + "github.com/go-ruby-parser/parser/token" +) + +// TestNegateLiteralRefusesANonNumericNode covers negateLiteral's default arm, +// which no Ruby source can reach: parseUnary calls the helper only after +// parsePrimary consumed an INT or FLOAT token, so the node is always an IntLit, a +// BignumLit or a FloatLit, possibly wrapped by applyNumSuffix in a RationalLit +// and/or an ImaginaryLit — the five arms above it. +// +// The arm is kept rather than deleted because deleting it means going back to the +// trailing `n.(*ast.FloatLit)` assertion it replaced, which panicked on every +// `-3r`, `-3i` and `-3ri`. Its job is to preserve the parser's never-panics +// contract if some future caller reaches here with another node kind: a refusal, +// not an assertion. This white-box test is how that job is witnessed — the same +// reason parseHook exists in robustness_internal_test.go. +func TestNegateLiteralRefusesANonNumericNode(t *testing.T) { + // p.fail reads p.cur() for the line number, and the cursor clamps at the + // trailing EOF, so a one-token stream is the minimum valid state. + p := &Parser{toks: []token.Token{{Type: token.EOF}}} + var err error + func() { + defer func() { + r := recover() + if r == nil { + t.Fatal("negateLiteral(NilLit): returned normally, want a parseError") + } + pe, ok := r.(parseError) + if !ok { + t.Fatalf("negateLiteral(NilLit): panicked with %T (%v), want a parseError", r, r) + } + err = pe + }() + p.negateLiteral(&ast.NilLit{}) + }() + if !strings.Contains(err.Error(), "cannot negate") { + t.Errorf("error = %q, want it to name the refusal", err) + } +} diff --git a/parser.go b/parser.go index 8671718..3bcad0f 100644 --- a/parser.go +++ b/parser.go @@ -216,6 +216,19 @@ type Parser struct { // lines collects the 1-based source line each statement node began on; it // becomes ast.Program.Lines. lines map[ast.Node]int + // stmtDepth is the nesting of statement LISTS: parseStatements increments it, + // so depth 1 is the program's own list. It exists for `BEGIN { }`, which MRI + // reaches only from `top_stmt` (parse.y v3_4_0 3020) — every nested body goes + // through `stmt_or_begin` (3093), whose `keyword_BEGIN` alternative exists + // solely to say "BEGIN is permitted only at toplevel". A depth is enough + // because MRI's own nesting rule is the same shape: `begin_block` reads a + // `top_compstmt` (3029), so a BEGIN inside a BEGIN is at top level again, which + // parseExeBlock spells by lending its body the enclosing depth. + stmtDepth int + // preExe collects the statements of every `BEGIN { }` block. MRI accumulates + // them in `p->eval_tree_begin` and runs them before the program body; Parse + // prepends them for the same reason, in source order. + preExe []ast.Node } // parseHook, when non-nil, runs at the start of Parse. It exists only so a @@ -257,6 +270,11 @@ func Parse(src string) (prog *ast.Program, err error) { } body := p.parseStatements(map[token.Type]bool{}) p.expect(token.EOF) + // Every `BEGIN { }` block runs before the program body (MRI's + // `p->eval_tree_begin`, prepended in the `program` rule), in source order. + if len(p.preExe) > 0 { + body = append(p.preExe, body...) + } return &ast.Program{Body: body, Lines: p.lines}, nil } @@ -562,9 +580,11 @@ func (p *Parser) parseStatements(stop map[token.Type]bool) []ast.Node { // that do reset it clear it themselves (parseBlockRest, parseLambda's `{` body). savedDoBlock := p.noDoBlock p.noDoBlock = false + p.stmtDepth++ defer func() { p.noMasgn = savedMasgn p.noDoBlock = savedDoBlock + p.stmtDepth-- }() var body []ast.Node for { @@ -617,11 +637,69 @@ func (p *Parser) parseStatement1() ast.Node { return p.applyModifiers(p.parseAlias()) case token.UNDEF: return p.applyModifiers(p.parseUndef()) + case token.CONST: + // `BEGIN { … }` and `END { … }`. Both spellings are reserved words in MRI + // (`reswords` lists keyword_BEGIN and keyword_END, parse.y v3_4_0 3819), but + // they are the only two that read as ordinary constants everywhere else, so + // they arrive here as a CONST token and are recognised by a hugging `{`. + // `do…end` is not an alternative in either production: MRI answers + // "unexpected 'do', expecting '{'". + if p.cur().Lit == "BEGIN" || p.cur().Lit == "END" { + if p.peekTok().Type == token.LBRACE { + return p.parseExeBlock() + } + } + return p.applyModifiers(p.parseOneLineMatch(p.parseKeywordLogical())) default: return p.applyModifiers(p.parseOneLineMatch(p.parseKeywordLogical())) } } +// parseExeBlock parses `BEGIN { body }` (parse.y v3_4_0 3020 `top_stmt: +// keyword_BEGIN begin_block`, 3029 `begin_block: '{' top_compstmt '}'`) or +// `END { body }` (3193 `stmt: k_END allow_exits '{' compstmt '}'`). +// +// BEGIN runs before the program body, which MRI does by accumulating the block +// into `p->eval_tree_begin` and leaving a void `NEW_BEGIN(0)` where it stood. +// Both halves are reproduced: the body is appended to p.preExe (Parse prepends +// it) and an empty *ast.Begin — a begin with no body, which evaluates to nil and +// does nothing — is returned in its place. BEGIN introduces no scope of its own +// in MRI (its body is a `top_compstmt` in the enclosing one), and hoisting into +// the same Program.Body keeps that. +// +// END is desugared to `at_exit { body }`. That is the replacement MRI itself +// names (`rb_warn0("END in method; use at_exit")`, 3196) and it was measured +// equivalent for what a parser can affect: MRI compiles the body to a +// NEW_SCOPE2 with an empty local table, and on ruby 4.0.5 `x = 1; END { p x }` +// and `x = 1; at_exit { p x }` both print 1 — the enclosing locals are visible +// either way. The alternative, a dedicated AST node, would need every consumer +// to learn it for no measurable difference here. +func (p *Parser) parseExeBlock() ast.Node { + pre := p.advance().Lit == "BEGIN" // BEGIN or END + if pre && p.stmtDepth != 1 { + p.fail("BEGIN is permitted only at toplevel") + } + p.expect(token.LBRACE) + if pre { + // begin_block's body is a `top_compstmt`, so a BEGIN nested in a BEGIN is + // still at top level (`BEGIN { BEGIN { } }` parses in MRI): lend the body the + // enclosing depth rather than a deeper one. + p.stmtDepth-- + } + restore := p.enterDoScope() + body := p.parseStatements(braceBlockEnd) + restore() + if pre { + p.stmtDepth++ + } + p.expect(token.RBRACE) + if pre { + p.preExe = append(p.preExe, body...) + return &ast.Begin{} + } + return &ast.Call{Name: "at_exit", Block: &ast.Block{Body: body}} +} + // parseKeywordLogical parses the low-precedence keyword operators `and`, `or`, // and the prefix `not`. They bind looser than `=` (and everything else except // the trailing if/unless/while/until modifiers), so they sit between an @@ -743,6 +821,42 @@ func (p *Parser) parseConstPath() (name string, path ast.Node) { return name, scoped } +// checkSingletonRecv refuses a `def` receiver that names a literal. MRI does it +// in the `singleton` production itself, with the message this one copies, over a +// fixed list of node types: NODE_STR, DSTR, XSTR, DXSTR, REGX, DREGX, SYM, DSYM, +// LINE, FILE, ENCODING, INTEGER, FLOAT, RATIONAL, IMAGINARY, LIST and ZLIST +// (parse.y v3_4_0 6724-6744). Notably absent: hashes and nil/true/false/self, so +// `def (nil).m` and `def ({}).m` are accepted. +// +// Ruby 4.0 moved the check: v3_4_0 applies it only to the `'(' expr rparen` +// alternative, while 4.0.5 restructured `singleton` into +// `value_expr(singleton_expr)` and applies it to the bare `var_ref` spelling too. +// The pseudo-literals are the only shapes that distinguishes: `def __FILE__.m` +// parses under the v3_4_0 grammar and is refused by ruby 4.0.5 under BOTH its +// parsers. The oracle this repo measures against is 4.0.5, so the check covers +// both paths — named here once so the two call sites cannot drift apart. +// +// One shape is knowingly left accepted: an interpolated symbol `:"a#{b}"`, which +// this parser desugars to `"a#{b}".to_sym` (see parseSymbolLiteral). That is an +// *ast.Call indistinguishable from a real `"a#{b}".to_sym`, which MRI accepts as +// a receiver, so refusing on the shape would refuse a legal program. +func (p *Parser) checkSingletonRecv(n ast.Node) { + switch lit := n.(type) { + case *ast.IntLit, *ast.BignumLit, *ast.FloatLit, *ast.RationalLit, *ast.ImaginaryLit, + *ast.StringLit, *ast.StrInterp, *ast.SymbolLit, *ast.RegexpLit, *ast.XStr, *ast.ArrayLit: + p.fail("can't define singleton method for literals") + case *ast.VarRef: + // __FILE__, __LINE__ and __ENCODING__ are `keyword_variable`s in MRI, so they + // reach the receiver position as themselves; here the lexer yields plain + // identifiers. Matching the name is sound because none of the three can ever + // be a local: they are reserved words, so no assignment can introduce them. + switch lit.Name { + case "__FILE__", "__LINE__", "__ENCODING__": + p.fail("can't define singleton method for literals") + } + } +} + func (p *Parser) parseClass() ast.Node { p.expect(token.CLASS) // `class << target` opens target's singleton (metaclass). A SHOVEL here is @@ -790,15 +904,29 @@ func (p *Parser) parseModule() ast.Node { // isDefRecvStart reports whether tt can begin an explicit `def` receiver // (`def self.x`, `def obj.x`, `def Const.x`, `def @ivar.x`, `def @@c.x`, -// `def $g.x`). +// `def $g.x`, `def nil.x`). MRI's receiver is `singleton: var_ref | '(' expr +// rparen` (parse.y v3_4_0 6711), and `var_ref: user_variable | keyword_variable` +// — one token, never a constant PATH, which is why `def M::Bar::baz` is a +// SyntaxError in MRI. `keyword_variable` (6711 ff.) is nil/self/true/false and +// the three pseudo-literals __FILE__/__LINE__/__ENCODING__; the first four are +// accepted here, the pseudo-literals deliberately are not (see parseDef). func isDefRecvStart(tt token.Type) bool { switch tt { - case token.SELF, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR: + case token.SELF, token.IDENT, token.CONST, token.IVAR, token.CVAR, token.GVAR, + token.NIL, token.TRUE, token.FALSE: return true } return false } +// isDefRecvSep reports whether tt separates a `def` receiver from the method +// name. MRI spells it `dot_or_colon: '.' | tCOLON2` (parse.y v3_4_0 6839) and +// `defs_head: k_def singleton dot_or_colon def_name` (3405) accepts either, so +// `def M::name(v)` and `def self::m` are the same production as `def M.name(v)` +// and `def self.m`. `&.` is NOT in it: `dot_or_colon` omits the `call_op` +// alternative tANDDOT, so `def obj&.m` is a SyntaxError. +func isDefRecvSep(tt token.Type) bool { return tt == token.DOT || tt == token.SCOPE } + func (p *Parser) parseDef() ast.Node { p.expect(token.DEF) singleton := false @@ -822,13 +950,19 @@ func (p *Parser) parseDef() ast.Node { p.fail("singleton def receiver must be a single expression") } recv = stmts[0] + p.checkSingletonRecv(recv) p.expect(token.RPAREN) - p.expect(token.DOT) + if !isDefRecvSep(p.cur().Type) { + p.fail("expected . or :: after a parenthesised def receiver") + } + p.advance() // . or :: } // A receiver before the method name: def self.foo / def obj.foo / def Const.foo - // / def @ivar.foo / def $g.foo. The kind guard keeps peekTok in range (the - // receiver is always a single name token followed by a dot). - if recv == nil && isDefRecvStart(p.cur().Type) && p.peekTok().Type == token.DOT { + // / def @ivar.foo / def $g.foo / def nil.foo, and each of those spelled with + // `::` instead of `.` (`def M::name(v)`, `def self::m`) — one `dot_or_colon` + // production, see isDefRecvSep. The kind guard keeps peekTok in range (the + // receiver is always a single name token followed by the separator). + if recv == nil && isDefRecvStart(p.cur().Type) && isDefRecvSep(p.peekTok().Type) { switch p.cur().Type { case token.SELF: p.advance() // self @@ -843,9 +977,21 @@ func (p *Parser) parseDef() ast.Node { recv = &ast.CVarRef{Name: p.advance().Lit} case token.GVAR: recv = &ast.GVarRef{Name: p.advance().Lit} + case token.NIL: // keyword_variable receivers: MRI parses them, then raises + p.advance() + recv = &ast.NilLit{} + case token.TRUE: + p.advance() + recv = &ast.BoolLit{Value: true} + case token.FALSE: + p.advance() + recv = &ast.BoolLit{Value: false} } if recv != nil || singleton { - p.advance() // . + if recv != nil { + p.checkSingletonRecv(recv) + } + p.advance() // . or :: } } name, ok := p.parseDefName() @@ -1671,7 +1817,13 @@ func (p *Parser) parseInterpBody() ast.Node { func (p *Parser) parseCase() ast.Node { p.expect(token.CASE) var subject ast.Node - if !p.is(token.NEWLINE) { + // `k_case terms? case_body k_end` (parse.y v3_4_0 6136) is the subject-less + // `case`, and its `terms?` is OPTIONAL — so the first clause keyword may follow + // `case` with no terminator at all (`case when x then … end`), not only after a + // newline. That shape also arrives here once the lexer joins `case`⏎`when …` + // (see isContinuationOp), which is why the guard names the keywords rather than + // just the newline. + if !p.is(token.NEWLINE) && !p.is(token.WHEN) && !p.is(token.IN) { // `k_case expr_value opt_terms case_body k_end` — the subject admits a // command (`case foo bar` ⏎ `when 1` ⏎ `end` parses in MRI) but, being an // `expr_value`, not a modifier-`rescue`: `case x rescue nil; when 1; end` @@ -2923,20 +3075,36 @@ func (p *Parser) inlineOpAssign(node ast.Node) ast.Node { } // negateLiteral returns the negation of a numeric literal node. The MINUS path -// in parseUnary reaches here only after parsePrimary consumed an INT or FLOAT -// token, which yields exactly one of these three kinds: a FLOAT is always a -// FloatLit, and an INT is an IntLit or — when it overflows int64, e.g. -// -9999999999999999999999999999999 — a BignumLit. (An INT with invalid digits -// fails inside parsePrimary and never returns here.) -func negateLiteral(n ast.Node) ast.Node { +// in parseUnary reaches here after parsePrimary consumed an INT or FLOAT token, +// so the node is an IntLit, a BignumLit (an INT that overflows int64, e.g. +// -9999999999999999999999999999999), or a FloatLit — each possibly wrapped by +// applyNumSuffix in a RationalLit and/or an ImaginaryLit for an `r`/`i` suffix +// (`-3r`, `-3i`, `-2.5ri`). Those wrappers are why this is a full switch with a +// default arm and not a trailing `n.(*ast.FloatLit)` assertion: that assertion +// panicked on every negated `r`/`i` literal, and the default keeps the parser's +// never-panics contract for a node kind that does not belong here by refusing +// the input instead of asserting. +// +// Negating through the wrapper matches MRI, which sets a `minus` flag on the +// OUTERMOST node (`negate_lit`, parse.y v3_4_0 — NODE_RATIONAL and NODE_IMAGINARY +// each carry their own `minus`). For these shapes negating the innermost numeric +// is the same number: (-3)/1 is (-3/1), and 0 + (-3/1)i is (0-(3/1)*i), which is +// exactly what `p(-3ri)` prints. +func (p *Parser) negateLiteral(n ast.Node) ast.Node { switch lit := n.(type) { case *ast.IntLit: return &ast.IntLit{Value: -lit.Value} case *ast.BignumLit: return &ast.BignumLit{Val: new(big.Int).Neg(lit.Val)} - } - fl := n.(*ast.FloatLit) - return &ast.FloatLit{Value: -fl.Value} + case *ast.FloatLit: + return &ast.FloatLit{Value: -lit.Value} + case *ast.RationalLit: + return &ast.RationalLit{Value: p.negateLiteral(lit.Value)} + case *ast.ImaginaryLit: + return &ast.ImaginaryLit{Value: p.negateLiteral(lit.Value)} + } + p.fail("cannot negate a non-numeric literal") + return nil } func (p *Parser) parseUnary() ast.Node { @@ -2952,7 +3120,7 @@ func (p *Parser) parseUnary() ast.Node { right := p.parseBinary(binBP(token.POW) - 1) return &ast.UnaryExpr{Op: "-", Operand: &ast.BinaryExpr{Op: "**", Left: lit, Right: right}} } - return p.parsePostfixTail(negateLiteral(lit)) + return p.parsePostfixTail(p.negateLiteral(lit)) } // `arg: tUMINUS arg` — an `arg` operand, so `-foo bar` is a SyntaxError in // MRI. `!` and `not` are the exception: they have an `expr: '!' command_call` @@ -3243,12 +3411,26 @@ func (p *Parser) parseLambda() ast.Node { restore() p.expect(token.RPAREN) p.scope().explicitParams = true - } else if p.is(token.IDENT) || p.is(token.STAR) || p.is(token.POW) || p.is(token.AMPER) { + } else if p.is(token.IDENT) || p.is(token.LABEL) || p.is(token.STAR) || p.is(token.POW) || p.is(token.AMPER) { // Unparenthesized parameters: `->x { }`, `-> ctx { }`, `->a, b { }`, - // `-> message do … end`. The list runs up to the block opener (`{`/`do`); - // parseBlockParams stops at the first token that is not a parameter. A `{` - // reached at this nesting is the lambda body, not a block on whatever the - // last default expression called: `-> a=a() { a }` (see noBraceBlock). + // `-> message do … end`, and — the case this set used to miss — a list that + // BEGINS with a keyword parameter: `-> x: 1 { x }`, `-> x: { x }`. + // + // The paren-less form is the plain `f_larglist: f_args` alternative (parse.y + // v3_4_0 5216), i.e. the SAME `f_args` the parenthesised form takes, and + // `f_args: args_tail` (6473) makes a bare `args_tail` a complete parameter + // list. `args_tail` starts with `f_kwarg(f_kw)` (6371), `f_kw: f_label + // arg_value | f_label` (6611) and `f_label: tLABEL` (6591) — so a tLABEL is + // a legal FIRST token here, exactly as `**kw` (f_any_kwrest) and `&b` + // (f_block_arg) already were. Omitting it made the whole list invisible, so + // `-> x: 1 { x }` was read as `->` followed immediately by a body and failed + // on "expected {"; `-> x, y: 2 { }` parsed only because a positional came + // first and parseBlockParams then handled the label itself. + // + // The list runs up to the block opener (`{`/`do`); parseBlockParams stops at + // the first token that is not a parameter. A `{` reached at this nesting is + // the lambda body, not a block on whatever the last default expression + // called: `-> a=a() { a }` (see noBraceBlock). savedBrace := p.noBraceBlock p.noBraceBlock = true params, defaults, prepends, splat, blockParam = p.parseBlockParams(token.LBRACE) @@ -3411,15 +3593,27 @@ func (p *Parser) parseBlockParams(until token.Type) (names []string, defaults, p } break } - if p.accept(token.POW) { // **rest keyword-splat / anonymous ** (lambda params) + if p.accept(token.POW) { // **rest keyword-splat / anonymous ** / **nil // parseBlockParams folds a keyword-splat into the splat slot using the // sentinel name "**rest"/"**"; a consumer treats it as the kwrest. (Block // kwargs are rare; record it as a trailing rest so the shape round-trips.) - if p.is(token.IDENT) { + switch { + case p.is(token.IDENT): name := p.advance().Lit p.declareLocal(name) names = append(names, "**"+name) - } else { + case p.is(token.NIL): + // `{ |**nil| }` / `-> (**nil) { }` — the block list takes the same + // "accepts no keywords" marker a method list does: MRI reaches it through + // `block_args_tail: f_any_kwrest opt_f_block_arg` (parse.y v3_4_0 5006), + // `f_any_kwrest: f_kwrest | f_no_kwarg` and `f_no_kwarg: p_kwnorest: + // kwrest_mark keyword_nil`. parseDefParams already spelled it "nil"; + // here the sentinel keeps the "**" prefix the rest of this list uses, so + // it cannot collide with a parameter actually named `nil` (there can be + // none — `nil` is a keyword). + p.advance() + names = append(names, "**nil") + default: names = append(names, "**") } defaults = append(defaults, nil)