From 624b4ccc159342641b57f47c3398c76f6268652e Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:44:27 +0200 Subject: [PATCH 01/10] Lex ?x character literals A in value position (or command-argument position) followed by a single character or backslash escape is a one-character String literal (MRI: ), distinct from the ternary that follows a finished value. Escapes mirror double-quote semantics; a multi-byte UTF-8 rune is taken whole. Co-Authored-By: Claude Opus 4.8 --- lexer/lexer.go | 100 +++++++++++++++++++++++++++++++++++++ round3_features_test.go | 107 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 round3_features_test.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 9ff337f..cfc7f83 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -150,6 +150,12 @@ func (l *Lexer) lexToken() token.Token { case c == '%' && l.prevType != token.DEF && l.percentBeginsLiteral(spaceBefore) && l.atPercentString(): // %q(…) / %Q(…) / %(…) / %W(…) string literals. return l.lexPercentString(spaceBefore, line, col) + case c == '?' && l.charLiteralBegins(spaceBefore) && l.atCharLiteral(): + // A `?x` character literal: a `?` in value position followed by a single + // character (or backslash escape) that does not run into an identifier. It + // denotes the one-character String "x" (MRI: `?a == "a"`). The ternary `?` + // only arises after a value (exprEnd) where it is not a command argument. + return l.lexCharLiteral(spaceBefore, line, col) case c == '\n' || c == ';': l.advance() l.state = exprBegin @@ -908,6 +914,100 @@ func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token { return token.Token{Type: token.REGEXP, Lit: string(src), Flags: string(flags), Line: line, Col: col, SpaceBefore: spaceBefore} } +// charLiteralBegins reports whether a `?` may open a character literal here, +// mirroring the regexp/percent rule: at expression-begin a value is expected, and +// in command-argument position (just after a value, but with a space before the +// `?` and the payload hugging it) the `?x` is the argument — `p ?a` is `p("a")` +// while `x ? y : z` (spaces around `?`) stays ternary. +func (l *Lexer) charLiteralBegins(spaceBefore bool) bool { + if l.state == exprBegin { + return true + } + // Command-argument position: just after a bareword that may be a method name + // (an IDENT/CONST), with a space before the `?` and the payload hugging it. + // This mirrors MRI's EXPR_CMDARG: `p ?a` is `p("a")`, while after a literal / + // `)` / `]` (a finished value) `?` is ternary even with surrounding spaces. + return spaceBefore && (l.prevType == token.IDENT || l.prevType == token.CONST) +} + +// atCharLiteral reports whether a `?` at the cursor (known to be in value +// position) opens a `?x` character literal rather than the ternary operator. A +// char literal needs a single character payload: either a `\`-escape (`?\n`, +// `?\s`, `?\101`), or one character (possibly a multi-byte UTF-8 rune) that is +// not whitespace and is not the start of a longer word — i.e. an alphanumeric +// payload must not be followed by another identifier byte (`?ab` is not a char +// literal; `?a.upcase` is `?a` then `.upcase`). +func (l *Lexer) atCharLiteral() bool { + n := l.peek2() + if n == 0 || n == ' ' || n == '\t' || n == '\n' || n == '\r' || n == '\f' || n == '\v' { + return false + } + if n == '\\' { // an escape always forms a char literal (?\n, ?\s, ?\\) + return true + } + // A plain single byte that begins an identifier must stand alone: the byte + // after it must not continue an identifier word (so `?a` is a char but `?ab` + // is not). A non-identifier payload (`?|`, `?/`, `?.`) is always a char. + if isIdentPart(n) { + third := byte(0) + if l.pos+2 < len(l.src) { + third = l.src[l.pos+2] + } + return !isIdentPart(third) + } + // A multi-byte UTF-8 lead byte (>= 0x80) starts a single rune payload (`?é`). + if n >= 0x80 { + return true + } + return true +} + +// lexCharLiteral lexes a `?x` character literal (cursor on the `?`), producing a +// STRING token holding the single character. It resolves the same backslash +// escapes a double-quoted string does; a multi-byte UTF-8 rune is taken whole. +func (l *Lexer) lexCharLiteral(spaceBefore bool, line, col int) token.Token { + l.advance() // '?' + var val string + if l.peek() == '\\' { + l.advance() // backslash + esc := l.advance() + switch esc { + case 'n': + val = "\n" + case 't': + val = "\t" + case 'r': + val = "\r" + case 's': + val = " " + case 'a': + val = "\x07" + case 'b': + val = "\x08" + case 'v': + val = "\x0b" + case 'f': + val = "\x0c" + case 'e': + val = "\x1b" + case '0': + val = "\x00" + default: + val = string(esc) + } + } else { + // One character: a full UTF-8 rune (its continuation bytes are >= 0x80). + var b []byte + b = append(b, l.advance()) + for l.peek() >= 0x80 { + b = append(b, l.advance()) + } + val = string(b) + } + l.state = exprEnd + return token.Token{Type: token.STRING, Lit: val, Line: line, Col: col, SpaceBefore: spaceBefore} +} + // percentDelimClose returns the closing delimiter for a %-literal opener: the // mate of a bracket pair, or the same character for a symmetric delimiter. func percentDelimClose(open byte) byte { diff --git a/round3_features_test.go b/round3_features_test.go new file mode 100644 index 0000000..3f8405c --- /dev/null +++ b/round3_features_test.go @@ -0,0 +1,107 @@ +package parser_test + +import ( + "testing" + + "github.com/go-ruby-parser/parser" + "github.com/go-ruby-parser/parser/ast" +) + +// mustParse parses src, failing the test on error. +func mustParse(t *testing.T, src string) *ast.Program { + t.Helper() + prog, err := parser.Parse(src) + if err != nil { + t.Fatalf("Parse(%q): %v", src, err) + } + return prog +} + +// --- Feature 6: character literals ?x --- + +// charLitValue digs the single-character StringLit out of `p ?x` (a Call whose +// sole argument is the literal). +func charLitArg(t *testing.T, src string) string { + t.Helper() + call, ok := mustParseSingle(t, src).(*ast.Call) + if !ok { + t.Fatalf("Parse(%q): top node is not a Call", src) + } + if len(call.Args) != 1 { + t.Fatalf("Parse(%q): want 1 arg, got %d", src, len(call.Args)) + } + s, ok := call.Args[0].(*ast.StringLit) + if !ok { + t.Fatalf("Parse(%q): arg is %T, want *ast.StringLit", src, call.Args[0]) + } + return s.Value +} + +func TestCharLiteralBasic(t *testing.T) { + cases := map[string]string{ + `p ?a`: "a", + `p ?|`: "|", + `p ?/`: "/", + `p ?Z`: "Z", + `p ?1`: "1", + `p ?.`: ".", + "p ?\\n": "\n", + "p ?\\t": "\t", + "p ?\\s": " ", + "p ?\\\\": "\\", + "p ?\\e": "\x1b", + "p ?\\0": "\x00", + `p ?é`: "é", + } + for src, want := range cases { + if got := charLitArg(t, src); got != want { + t.Errorf("%q: char value = %q, want %q", src, got, want) + } + } +} + +func TestCharLiteralAtExprBegin(t *testing.T) { + asn := mustParseSingle(t, `x = ?a`).(*ast.Assign) + s, ok := asn.Value.(*ast.StringLit) + if !ok || s.Value != "a" { + t.Fatalf("x = ?a: RHS = %#v, want StringLit \"a\"", asn.Value) + } +} + +func TestCharLiteralWithPostfix(t *testing.T) { + // `p ?a.upcase` is `p((?a).upcase)`, not a ternary. + call := mustParseSingle(t, `p ?a.upcase`).(*ast.Call) + inner, ok := call.Args[0].(*ast.Call) + if !ok || inner.Name != "upcase" { + t.Fatalf("p ?a.upcase: arg = %#v, want .upcase call", call.Args[0]) + } + if s, ok := inner.Recv.(*ast.StringLit); !ok || s.Value != "a" { + t.Fatalf("p ?a.upcase: receiver = %#v, want StringLit \"a\"", inner.Recv) + } +} + +func TestQuestionStaysTernaryAfterValue(t *testing.T) { + // After a finished value the `?` is the ternary operator, never a char + // literal, even when a digit hugs it: `p 1 ?2 :3` == `p(1 ? 2 : 3)`. + call := mustParseSingle(t, `p 1 ?2 :3`).(*ast.Call) + if _, ok := call.Args[0].(*ast.If); !ok { + t.Fatalf("p 1 ?2 :3: arg = %T, want a ternary (*ast.If)", call.Args[0]) + } +} + +func TestTernaryWithSpacedQuestion(t *testing.T) { + // `x ? y : z` over locals stays a ternary. + prog := mustParse(t, "x=1\ny=2\np x ? y : 0") + call := prog.Body[2].(*ast.Call) + if _, ok := call.Args[0].(*ast.If); !ok { + t.Fatalf("x ? y : 0: arg = %T, want a ternary (*ast.If)", call.Args[0]) + } +} + +func TestQuestionNotCharLiteralForWord(t *testing.T) { + // `?ab` is not a char literal (two chars) — MRI rejects it; here `?` after a + // value position with a word also is not a single-char literal. + if _, err := parser.Parse(`p ?ab`); err == nil { + t.Fatalf("p ?ab: expected a parse error (not a char literal)") + } +} From 0cc053f6d1166007c4213f7e75c6eb8bee74d2f1 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:46:28 +0200 Subject: [PATCH 02/10] Generalize percent-literal delimiters to any non-alphanumeric char MRI accepts any non-alphanumeric, non-whitespace character as a percent-literal delimiter for every percent kind: %r"...", %Q'...', %q[...], %w'a b', %i, %(...), %!...!, %|...|, %@...@, etc. isPercentDelim now admits the full set (excluding alphanumerics, whitespace, multibyte leads, and '=' so a %= compound assignment is unaffected); %w/%i delimiter detection shares it. Co-Authored-By: Claude Opus 4.8 --- lexer/lexer.go | 31 +++++++++++++----------- round3_features_test.go | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index cfc7f83..be91306 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1046,14 +1046,7 @@ func (l *Lexer) atPercentArray() bool { default: return false } - if l.pos+2 >= len(l.src) { - return false - } - switch l.src[l.pos+2] { - case '[', '(', '{', '<', '|', '!', '/': - return true - } - return false + return l.pos+2 < len(l.src) && isPercentDelim(l.src[l.pos+2]) } // lexPercentArray lexes a %w/%i/%W/%I array literal. The non-interpolating @@ -1094,13 +1087,25 @@ func (l *Lexer) lexPercentArray(spaceBefore bool, line, col int) token.Token { return token.Token{Type: tt, Lit: string(content), Line: line, Col: col, SpaceBefore: spaceBefore} } -// isPercentDelim reports whether b can open a %-literal. +// isPercentDelim reports whether b can open a %-literal. MRI accepts any +// non-alphanumeric character as the delimiter (`%r"..."`, `%q[...]`, `%w'a b'`, +// `%i`, `%(...)`, `%!...!`, `%|...|`, `%@...@`, …); a bracket pair nests. +// Excluded: alphanumerics (they would be the literal's body), whitespace, and +// `=` so a `%=` compound assignment is never mistaken for a literal opener. func isPercentDelim(b byte) bool { - switch b { - case '(', '[', '{', '<', '|', '!', '/': - return true + switch { + case b == 0 || b == '=': + return false + case b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\f' || b == '\v': + return false + case b >= '0' && b <= '9': + return false + case (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'): + return false + case b >= 0x80: + return false // a multi-byte UTF-8 lead is not used as a %-delimiter here } - return false + return true } // atPercentRXS reports whether the cursor (at '%') begins a %r (regexp), diff --git a/round3_features_test.go b/round3_features_test.go index 3f8405c..5606087 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -105,3 +105,55 @@ func TestQuestionNotCharLiteralForWord(t *testing.T) { t.Fatalf("p ?ab: expected a parse error (not a char literal)") } } + +// --- Feature 7: generalized percent-literal delimiters --- + +func TestPercentDelimiterVariants(t *testing.T) { + // Any non-alphanumeric delimiter opens the literal; the result node is the + // same as the bracket-delimited form. + for _, src := range []string{ + `%r"abc"`, `%r|ab|`, `%r#ab#`, + `%Q'hi'`, `%Q@hi@`, `%Q!hi!`, + `%q[x]`, `%q@x@`, `%q'x'`, + `%w'a b'`, `%w*a b*`, `%w|a b|`, + `%i`, `%i!x y!`, + `%W{a b}`, `%I`, + `%(hi)`, `%!hi!`, `%@hi@`, + `%s"sym"`, `%s|sym|`, + `%x"echo hi"`, `%x|ls|`, + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +func TestPercentArrayQuoteDelim(t *testing.T) { + arr := mustParseSingle(t, `%w'apple pear'`).(*ast.ArrayLit) + if len(arr.Elems) != 2 { + t.Fatalf(`%%w'apple pear': got %d elems, want 2`, len(arr.Elems)) + } + if s, ok := arr.Elems[0].(*ast.StringLit); !ok || s.Value != "apple" { + t.Fatalf("first elem = %#v, want StringLit \"apple\"", arr.Elems[0]) + } +} + +func TestPercentRegexpQuoteDelim(t *testing.T) { + re := mustParseSingle(t, `%r"a.c"i`).(*ast.RegexpLit) + if re.Source != "a.c" || re.Flags != "i" { + t.Fatalf(`%%r"a.c"i: source=%q flags=%q, want "a.c"/"i"`, re.Source, re.Flags) + } +} + +func TestPercentDoesNotBreakModuloOrOpAssign(t *testing.T) { + // `%=` stays a compound assignment; `a % 3` stays modulo. + prog := mustParse(t, "a = 10\na %= 3") + if _, ok := prog.Body[1].(*ast.OpAssign); !ok { + t.Fatalf("a %%= 3: node = %T, want *ast.OpAssign", prog.Body[1]) + } + prog = mustParse(t, "a = 10\np a % 3") + call := prog.Body[1].(*ast.Call) + if _, ok := call.Args[0].(*ast.BinaryExpr); !ok { + t.Fatalf("a %% 3: arg = %T, want *ast.BinaryExpr (modulo)", call.Args[0]) + } +} From 150fad89d7053544425fc37d84aa6b909c9dc8c0 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:51:20 +0200 Subject: [PATCH 03/10] Fold adjacent string literals and accept string-hugging command args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjacent string literals concatenate at parse time across every form, including interpolated pieces: "a" "b" folds to one StringLit, "a" "b#{x}" folds to one StrInterp, and a backslash-continued line joins too. A string literal that hugs a name (no intervening space) is a paren-less command argument — foo"bar", step"x" do…end, obj.m"x", Foo"bar" — read as a call even when the name is a local (MRI: x"y" is x("y")). Co-Authored-By: Claude Opus 4.8 --- parser.go | 101 +++++++++++++++++++++++++++++++++++----- round3_features_test.go | 85 +++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+), 12 deletions(-) diff --git a/parser.go b/parser.go index 8901640..9996f37 100644 --- a/parser.go +++ b/parser.go @@ -860,6 +860,57 @@ func (p *Parser) parseBodyWithRescue() []ast.Node { return body } +// parseStringConcat parses one or more adjacent string literals and folds them +// into a single node, as MRI concatenates them at parse time: `"a" "b"` == `"ab"`, +// and a `\`-continued line joins too (the lexer already swallows the backslash + +// newline as whitespace). Each piece is a plain STRING or an interpolated +// STRBEG…STREND run. With a single plain piece it returns a StringLit; otherwise +// the pieces' parts are concatenated into one StrInterp (a run that turns out to +// be all-plain is collapsed back to a StringLit). +func (p *Parser) parseStringConcat() ast.Node { + var pieces []ast.Node + for p.is(token.STRING) || p.is(token.STRBEG) { + if p.is(token.STRING) { + pieces = append(pieces, &ast.StringLit{Value: p.advance().Lit}) + } else { + pieces = append(pieces, p.parseInterpString()) + } + } + if len(pieces) == 1 { + return pieces[0] + } + // Concatenate: flatten every piece's parts into one interpolation, then + // collapse to a plain StringLit when no interpolation survives. + var parts []ast.Node + for _, pc := range pieces { + switch n := pc.(type) { + case *ast.StringLit: + parts = append(parts, n) + case *ast.StrInterp: + parts = append(parts, n.Parts...) + } + } + if lit, ok := mergeStringParts(parts); ok { + return lit + } + return &ast.StrInterp{Parts: parts} +} + +// mergeStringParts returns a single StringLit when every part is a plain string +// literal (so an all-plain concatenation like `"a" "b"` yields `"ab"`), else +// (nil, false). +func mergeStringParts(parts []ast.Node) (*ast.StringLit, bool) { + var b strings.Builder + for _, pt := range parts { + s, ok := pt.(*ast.StringLit) + if !ok { + return nil, false + } + b.WriteString(s.Value) + } + return &ast.StringLit{Value: b.String()}, true +} + // parseInterpString assembles an interpolated string from the lexer's // STRBEG (literal …#{) / expr / STRMID (}…#{) / … / STREND (}…") sequence. func (p *Parser) parseInterpString() ast.Node { @@ -1739,9 +1790,10 @@ func (p *Parser) parsePostfixTail(node ast.Node) ast.Node { p.advance() args = p.parseCallArgs(token.RPAREN) p.expect(token.RPAREN) - } else if p.canStartCommandArg() { + } else if p.canStartCommandArg() || p.atHuggingStringArg() { // Paren-less command call on a receiver: `obj.foo bar`, - // `Fiber.yield 1`. The space-separated argument list terminates + // `Fiber.yield 1`, or a string hugging the method name + // (`obj.m"x"`). The space-separated argument list terminates // this postfix chain (it greedily consumes the rest). A trailing // `do…end` then binds to this command call (`obj.foo bar do … end`), // as MRI attaches it to the outermost command rather than to an @@ -2143,14 +2195,8 @@ func (p *Parser) parsePrimary() ast.Node { p.advance() f, _ := strconv.ParseFloat(t.Lit, 64) return &ast.FloatLit{Value: f} - case token.STRING: - p.advance() - // Adjacent string literals concatenate: `"a" "b"` == `"ab"`. - s := t.Lit - for p.is(token.STRING) { - s += p.advance().Lit - } - return &ast.StringLit{Value: s} + case token.STRING, token.STRBEG: + return p.parseStringConcat() case token.SYMBOL: p.advance() return &ast.SymbolLit{Name: t.Lit} @@ -2217,6 +2263,10 @@ func (p *Parser) parsePrimary() ast.Node { p.expect(token.RPAREN) return &ast.Call{Name: t.Lit, Args: args} } + // A string hugging the constant is a paren-less call: `Integer"42"`. + if p.atHuggingStringArg() { + return &ast.Call{Name: t.Lit, Args: p.parseCommandArgs()} + } return &ast.ConstRef{Name: t.Lit} case token.SCOPE: // Leading `::Name` — a top-level constant lookup (`::Foo`, `defined?(::Foo)`). @@ -2247,8 +2297,6 @@ func (p *Parser) parsePrimary() ast.Node { return p.parseWhile() case token.UNTIL: return p.parseUntil() - case token.STRBEG: - return p.parseInterpString() } return p.fail("unexpected token %q (%s)", t.Lit, t.Type) } @@ -2284,6 +2332,19 @@ func (p *Parser) parseIdentExpr() ast.Node { return &ast.Call{Name: name, Args: args} } + // A string literal hugging the name (no space) is a paren-less argument: + // `step"Ensure ..." do … end`, `assert"x", y`, `foo"a"`. MRI reads this as a + // command call even when the name is otherwise a local — `x"y"` is `x("y")`. + // So this is checked before the local-variable resolution below. + if p.huggingStringArg() { + p.advance() // name + call := &ast.Call{Name: name, Args: p.parseCommandArgs()} + if p.is(token.DO) && !p.noDo { + call.Block = p.parseDoBlock() + } + return call + } + // Known local variable → read. if p.is(token.IDENT) && p.isLocal(name) { p.advance() @@ -2317,6 +2378,22 @@ func (p *Parser) parseIdentExpr() ast.Node { return &ast.Call{Name: name} } +// huggingStringArg reports whether the token after the current name is a string +// literal that hugs it (no intervening space), making the name a command call +// with that string as its first argument: `foo"bar"`, `step"x" do…end`. A +// space-separated string is handled by the ordinary canStartCommandArg path. +func (p *Parser) huggingStringArg() bool { return isHuggingString(p.peekTok()) } + +// atHuggingStringArg is huggingStringArg for a name already consumed: the cursor +// itself is the would-be string argument (used after a `.method` in a postfix +// chain, `obj.m"x"`). +func (p *Parser) atHuggingStringArg() bool { return isHuggingString(p.cur()) } + +// isHuggingString reports whether t is a string literal with no preceding space. +func isHuggingString(t token.Token) bool { + return !t.SpaceBefore && (t.Type == token.STRING || t.Type == token.STRBEG) +} + // canStartCommandArg decides whether the current token begins a paren-less // argument list. This is the `foo -1` (call) vs `foo - 1` (subtraction) // disambiguation, driven by SpaceBefore. diff --git a/round3_features_test.go b/round3_features_test.go index 5606087..fa347ca 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -145,6 +145,91 @@ func TestPercentRegexpQuoteDelim(t *testing.T) { } } +// --- Feature 3: adjacent string-literal concatenation --- + +func TestAdjacentStringConcat(t *testing.T) { + s, ok := mustParseSingle(t, `"a" "b" "c"`).(*ast.StringLit) + if !ok || s.Value != "abc" { + t.Fatalf(`"a" "b" "c" = %#v, want StringLit "abc"`, mustParseSingle(t, `"a" "b" "c"`)) + } +} + +func TestAdjacentStringConcatSingleQuote(t *testing.T) { + s := mustParseSingle(t, `'x' 'y'`).(*ast.StringLit) + if s.Value != "xy" { + t.Fatalf(`'x' 'y' = %q, want "xy"`, s.Value) + } +} + +func TestAdjacentStringConcatBackslashContinued(t *testing.T) { + s := mustParseSingle(t, "\"a\" \\\n \"b\"").(*ast.StringLit) + if s.Value != "ab" { + t.Fatalf(`backslash-continued concat = %q, want "ab"`, s.Value) + } +} + +func TestAdjacentStringConcatWithInterp(t *testing.T) { + // A plain piece adjacent to an interpolated one folds into one StrInterp. + for _, src := range []string{ + `"a" "b#{1}"`, + `"a#{1}" "b"`, + `"a#{1}" "b#{2}"`, + } { + n := mustParseSingle(t, src) + if _, ok := n.(*ast.StrInterp); !ok { + t.Errorf("Parse(%q): node = %T, want *ast.StrInterp", src, n) + } + } +} + +// --- Feature 1: string-adjacent command argument (no space) --- + +func TestHuggingStringCommandArg(t *testing.T) { + for _, src := range []string{ + `foo"bar"`, + `p"hello"`, + `assert"x", y`, + `foo"a""b"`, + `foo"a#{1}"`, + } { + call, ok := mustParseSingle(t, src).(*ast.Call) + if !ok { + t.Errorf("Parse(%q): node = %T, want *ast.Call", src, mustParseSingle(t, src)) + continue + } + if len(call.Args) == 0 { + t.Errorf("Parse(%q): call has no args", src) + } + } +} + +func TestHuggingStringCommandArgWithBlock(t *testing.T) { + call := mustParseSingle(t, `step"Ensure something" do; end`).(*ast.Call) + if call.Name != "step" || call.Block == nil { + t.Fatalf("step\"...\" do…end: name=%q block=%v, want step + a block", call.Name, call.Block) + } + if s, ok := call.Args[0].(*ast.StringLit); !ok || s.Value != "Ensure something" { + t.Fatalf("step arg = %#v, want StringLit", call.Args[0]) + } +} + +func TestHuggingStringOnLocalIsCall(t *testing.T) { + // `x"y"` is a command call even though x is a local (MRI: x("y")). + prog := mustParse(t, "x = 1\nx\"y\"") + call, ok := prog.Body[1].(*ast.Call) + if !ok || call.Name != "x" { + t.Fatalf("x\"y\" = %#v, want a Call named x", prog.Body[1]) + } +} + +func TestHuggingStringOnReceiverAndConst(t *testing.T) { + for _, src := range []string{`obj.foo"bar"`, `Foo"bar"`} { + if _, ok := mustParseSingle(t, src).(*ast.Call); !ok { + t.Errorf("Parse(%q): node = %T, want *ast.Call", src, mustParseSingle(t, src)) + } + } +} + func TestPercentDoesNotBreakModuloOrOpAssign(t *testing.T) { // `%=` stays a compound assignment; `a % 3` stays modulo. prog := mustParse(t, "a = 10\na %= 3") From 155e85d302f556ef66d01121e7e3f730b2d8eeb9 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:53:37 +0200 Subject: [PATCH 04/10] Continue a line ending in a ternary separator colon A bare ':' is unambiguously the ternary separator at the lexer level (a symbol ':x' and a label 'x:' are distinct tokens), so a line ending in it is incomplete and joins the next line, completing the multi-line ternary 'cond ?''a :''b'. The '?' branch was already a continuation operator; this adds the ':' branch. Co-Authored-By: Claude Opus 4.8 --- lexer/lexer.go | 9 +++++---- round3_features_test.go | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index be91306..0d139eb 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -99,16 +99,17 @@ func (l *Lexer) next() token.Token { // isContinuationOp reports whether a line ending in token type t is incomplete, // so the trailing newline continues onto the next line rather than terminating // the statement. These are infix operators (plus comma and a trailing dot) that -// require a right-hand operand. Deliberately excluded as ambiguous: `|` and `&` -// (block params / block-pass), `^` (pattern pin), `:` (label/symbol/ternary), -// `<<` (heredoc opener). +// require a right-hand operand. A bare `:` is included: it is only ever the +// ternary separator here (a symbol `:x` and a label `x:` are their own tokens), +// so `expr ? x :``y` joins. Deliberately excluded as ambiguous: `|` and +// `&` (block params / block-pass), `^` (pattern pin), `<<` (heredoc opener). func isContinuationOp(t token.Type) bool { switch t { case token.PLUS, token.MINUS, token.STAR, token.POW, token.SLASH, token.PERCENT, token.EQ, token.EQQ, token.MATCH, token.NEQ, token.LT, token.GT, token.LE, token.GE, token.SPACESHIP, token.ANDAND, token.OROR, token.ASSIGN, token.OPASSIGN, token.COMMA, token.HASHROCKET, - token.DOT, token.SAFEDOT, token.QUESTION, + token.DOT, token.SAFEDOT, token.QUESTION, token.COLON, // Trailing low-precedence keyword operators and modifiers whose operand / // condition may sit on the next line: `... or\n fail`, `... and\n x`, // `do_it unless\n cond`, `foo if\n bar` (MRI joins these lines too). diff --git a/round3_features_test.go b/round3_features_test.go index fa347ca..08a49cf 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -145,6 +145,29 @@ func TestPercentRegexpQuoteDelim(t *testing.T) { } } +// --- Feature 2: multi-line ternary --- + +func TestMultiLineTernary(t *testing.T) { + for _, src := range []string{ + "a=1\nb=2\nc=3\nx = a ?\n b :\n c", + "a=1\nb=2\nc=3\nx = a ? b :\n c", + "a=1\nb=2\nc=3\nx = a ?\n b : c", + "x=1\ny=2\nz=3\np x ?\ny :\nz", + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +func TestMultiLineTernaryShape(t *testing.T) { + prog := mustParse(t, "a=1\nb=2\nc=3\nx = a ?\n b :\n c") + asn := prog.Body[3].(*ast.Assign) + if _, ok := asn.Value.(*ast.If); !ok { + t.Fatalf("multi-line ternary RHS = %T, want *ast.If", asn.Value) + } +} + // --- Feature 3: adjacent string-literal concatenation --- func TestAdjacentStringConcat(t *testing.T) { From 4bc52a74e530eb383bf64421b5afc69901cdabf9 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:55:12 +0200 Subject: [PATCH 05/10] Allow an assignment as a nested sub-expression operand MRI permits an assignment to sit inside a larger expression even though '=' binds looser than the binary operators: 'if a && b = c', 'while x && line = gets', 'r = a && @b = 1'. After parsing a binary operator's right operand, an '=' that follows an assignable target (local/ivar/cvar/gvar/const, or an attribute/index receiver call) is consumed as an assignment. '=='/'=>'/'=~' are distinct tokens, so only a real '=' is taken. Co-Authored-By: Claude Opus 4.8 --- parser.go | 57 ++++++++++++++++++++++++++++++++++++++++- round3_features_test.go | 42 ++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 9996f37..3ee0cf4 100644 --- a/parser.go +++ b/parser.go @@ -1712,11 +1712,66 @@ func (p *Parser) parseBinary(minBP int) ast.Node { if op == "**" { // exponentiation is right-associative rbp = bp - 1 } - right := p.parseBinary(rbp) + right := p.maybeInlineAssign(p.parseBinary(rbp)) left = &ast.BinaryExpr{Op: op, Left: left, Right: right} } } +// maybeInlineAssign turns a freshly-parsed operand into an assignment when an +// `=` follows it and it names an assignable target. This lets an assignment sit +// as a sub-expression inside a condition or larger expression — `if a && b = c`, +// `while x && line = gets` — which MRI permits even though `=` otherwise binds +// looser than the binary operators. (A top-level `lhs = rhs` is already handled +// in parseExprOrAssign; this covers the nested operand case.) `==`/`=>`/`=~` are +// their own tokens, so only a real assignment `=` is consumed here. +func (p *Parser) maybeInlineAssign(node ast.Node) ast.Node { + if !p.is(token.ASSIGN) { + return node + } + switch n := node.(type) { + case *ast.VarRef: + p.advance() + p.declareLocal(n.Name) + return &ast.Assign{Name: n.Name, Value: p.parseAssignRhs()} + case *ast.Call: + // A receiver-less, argument-less bare call is really a not-yet-seen local: + // `b = c` where b was unknown. Receiver attribute/index targets become the + // setter-call shape, matching parseExprOrAssign's handling. + if n.Recv == nil && len(n.Args) == 0 && n.Block == nil { + p.advance() + p.declareLocal(n.Name) + return &ast.Assign{Name: n.Name, Value: p.parseAssignRhs()} + } + if n.Recv != nil && n.Block == nil { + if n.Name == "[]" { + p.advance() + n.Name = "[]=" + n.Args = append(n.Args, p.parseExprOrAssign()) + return n + } + if len(n.Args) == 0 { + p.advance() + n.Name += "=" + n.Args = []ast.Node{p.parseExprOrAssign()} + return n + } + } + case *ast.IvarRef: + p.advance() + return &ast.IvarAssign{Name: n.Name, Value: p.parseAssignRhs()} + case *ast.CVarRef: + p.advance() + return &ast.CVarAssign{Name: n.Name, Value: p.parseAssignRhs()} + case *ast.GVarRef: + p.advance() + return &ast.GVarAssign{Name: n.Name, Value: p.parseAssignRhs()} + case *ast.ConstRef: + p.advance() + return &ast.ConstAssign{Name: n.Name, Value: p.parseAssignRhs()} + } + return node +} + // negateLiteral returns the negation of a numeric literal node. The MINUS path // in parseUnary reaches here only after parsePrimary consumed an INT or FLOAT // token, which yields exactly one of these three kinds: a FLOAT is always a diff --git a/round3_features_test.go b/round3_features_test.go index 08a49cf..31cc086 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -253,6 +253,48 @@ func TestHuggingStringOnReceiverAndConst(t *testing.T) { } } +// --- Feature 4: assignment inside a condition / expression --- + +func TestInlineAssignmentInCondition(t *testing.T) { + for _, src := range []string{ + `if a && b = c; end`, + `if ct && m = RE.match(s); end`, + `if a || b = c; end`, + `while x && line = gets; end`, + `r = a && @b = 1`, + `if a && @@c = 2; end`, + `if a && $g = 3; end`, + `if a && K = 4; end`, + `if a && h[k] = 5; end`, + `if a && o.attr = 6; end`, + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +func TestInlineAssignmentShape(t *testing.T) { + // `if a && b = c` is `if a && (b = c)`. + ifn := mustParseSingle(t, "if a && b = c; end").(*ast.If) + be := ifn.Cond.(*ast.BinaryExpr) + if be.Op != "&&" { + t.Fatalf("cond op = %q, want &&", be.Op) + } + if _, ok := be.Right.(*ast.Assign); !ok { + t.Fatalf("RHS of && = %T, want *ast.Assign", be.Right) + } +} + +func TestInlineAssignmentDoesNotEatComparisons(t *testing.T) { + // `==`, `=>`, `=~` must not be swallowed as assignments. + for _, src := range []string{`p(a == b)`, `p(a =~ b)`, `x == y`} { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + func TestPercentDoesNotBreakModuloOrOpAssign(t *testing.T) { // `%=` stays a compound assignment; `a % 3` stays modulo. prog := mustParse(t, "a = 10\na %= 3") From d3bb2c90fa37fba5b710e9319719092328cd4e0a Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:56:52 +0200 Subject: [PATCH 06/10] Accept a value-less control-flow jump as a binary operand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MRI allows a bare jump as the right operand of && / ||: 'x || return', 'expr && next', 'cond || break', 'begin; x || retry; …'. Only the argument-less form is valid in this position ('x || return 5' is a syntax error), so the binary right-operand path consumes return/break/next/retry only when value-less. Co-Authored-By: Claude Opus 4.8 --- parser.go | 30 ++++++++++++++++++++++++++++++ round3_features_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/parser.go b/parser.go index 3ee0cf4..7ac01fa 100644 --- a/parser.go +++ b/parser.go @@ -1712,11 +1712,41 @@ func (p *Parser) parseBinary(minBP int) ast.Node { if op == "**" { // exponentiation is right-associative rbp = bp - 1 } + // A value-less control-flow jump may be the right operand of `&&`/`||`: + // `x || return`, `expr && next`, `y || break` (MRI accepts the jump only + // without a value here — `x || return 5` is a syntax error). + if jump, ok := p.valuelessJumpOperand(); ok { + left = &ast.BinaryExpr{Op: op, Left: left, Right: jump} + return left + } right := p.maybeInlineAssign(p.parseBinary(rbp)) left = &ast.BinaryExpr{Op: op, Left: left, Right: right} } } +// valuelessJumpOperand parses a value-less control-flow jump (`return`, `break`, +// `next`, `retry`) sitting in operand position, returning its node and true. MRI +// accepts a bare jump as the right operand of `&&`/`||` (`x || return`, `expr && +// next`) but not one carrying a value (`x || return 5` is a syntax error), so +// only the argument-less forms are recognised here. +func (p *Parser) valuelessJumpOperand() (ast.Node, bool) { + switch p.cur().Type { + case token.RETURN: + p.advance() + return &ast.Return{}, true + case token.BREAK: + p.advance() + return &ast.Break{}, true + case token.NEXT: + p.advance() + return &ast.Next{}, true + case token.RETRY: + p.advance() + return &ast.Retry{}, true + } + return nil, false +} + // maybeInlineAssign turns a freshly-parsed operand into an assignment when an // `=` follows it and it names an assignable target. This lets an assignment sit // as a sub-expression inside a condition or larger expression — `if a && b = c`, diff --git a/round3_features_test.go b/round3_features_test.go index 31cc086..ad113bf 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -253,6 +253,40 @@ func TestHuggingStringOnReceiverAndConst(t *testing.T) { } } +// --- Feature 8: control-flow keyword as expression operand --- + +func TestControlFlowKeywordAsOperand(t *testing.T) { + for _, src := range []string{ + `def f; x || return; end`, + `[1].each { |x| x || next }`, + `[1].each { |x| x || break }`, + `while true; x || break; end`, + `a = b || return`, + `def f; x or return; end`, + `def f; x && return; end`, + `begin; x || retry; rescue; end`, + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +func TestControlFlowOperandRejectsValue(t *testing.T) { + // MRI: a value after the jump in operand position is a syntax error. + if _, err := parser.Parse(`def f; x || return 5; end`); err == nil { + t.Fatalf("x || return 5: expected a parse error") + } +} + +func TestControlFlowOperandShape(t *testing.T) { + ifn := mustParseSingle(t, "def f; x || return; end").(*ast.MethodDef) + be := ifn.Body[0].(*ast.BinaryExpr) + if _, ok := be.Right.(*ast.Return); !ok { + t.Fatalf("RHS of || = %T, want *ast.Return", be.Right) + } +} + // --- Feature 4: assignment inside a condition / expression --- func TestInlineAssignmentInCondition(t *testing.T) { From 9b2cf85e0e40f15e6263b37a4cebb41ae4cc71ea Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:58:20 +0200 Subject: [PATCH 07/10] End a value-less jump at a closing paren or bracket A bare return/break/next/retry now ends at ')' or ']', so a control-flow jump can sit inside a parenthesised group: '( x or next )[1]', '(set_param(p) and return) unless cur'. Previously the jump greedily consumed the closing delimiter as its value. The '(expr if cond)' modifier form already worked via the group's statement-sequence parse. Co-Authored-By: Claude Opus 4.8 --- parser.go | 5 ++++- round3_features_test.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/parser.go b/parser.go index 7ac01fa..a3e1b91 100644 --- a/parser.go +++ b/parser.go @@ -795,10 +795,13 @@ func (p *Parser) parseNext() ast.Node { // atStatementEnd reports whether the cursor is at a point where a value-less // return/break/next ends: a terminator, a block/body close (end, else, elsif, -// `}`), or a trailing modifier (if/unless/while/until). +// `}`, `)`, `]`), or a trailing modifier (if/unless/while/until). The closing +// `)`/`]` let a bare jump end a parenthesised/bracketed group — `( x or next )`, +// `[a or break]` — rather than swallowing the delimiter as its value. func (p *Parser) atStatementEnd() bool { switch p.cur().Type { case token.NEWLINE, token.EOF, token.END, token.ELSE, token.ELSIF, token.RBRACE, + token.RPAREN, token.RBRACKET, token.IF, token.UNLESS, token.WHILE, token.UNTIL: return true } diff --git a/round3_features_test.go b/round3_features_test.go index ad113bf..a576c9b 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -253,6 +253,23 @@ func TestHuggingStringOnReceiverAndConst(t *testing.T) { } } +// --- Feature 5: control-flow + modifier inside parens --- + +func TestControlFlowInsideParens(t *testing.T) { + for _, src := range []string{ + `(set_param(p) and return) unless cur`, + `[1].each { ( x or next )[1] }`, + `(expr if cond)`, + `(x if y) || z`, + `[1].each { (m.match(s) or next)[1] }`, + `begin; (x or retry); rescue; end`, + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + // --- Feature 8: control-flow keyword as expression operand --- func TestControlFlowKeywordAsOperand(t *testing.T) { From ef042220dcb9f657eccd856da624b41a2e8498f6 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 18:59:27 +0200 Subject: [PATCH 08/10] Drop retry from valid jump operands (MRI rejects x || retry) retry is not accepted by MRI as a binary operand or an or/and operand, so valuelessJumpOperand now covers only return/break/next; corrects the round-3 operand tests accordingly. Co-Authored-By: Claude Opus 4.8 --- parser.go | 11 ++++------- round3_features_test.go | 2 -- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/parser.go b/parser.go index a3e1b91..afb6c84 100644 --- a/parser.go +++ b/parser.go @@ -1728,10 +1728,10 @@ func (p *Parser) parseBinary(minBP int) ast.Node { } // valuelessJumpOperand parses a value-less control-flow jump (`return`, `break`, -// `next`, `retry`) sitting in operand position, returning its node and true. MRI -// accepts a bare jump as the right operand of `&&`/`||` (`x || return`, `expr && -// next`) but not one carrying a value (`x || return 5` is a syntax error), so -// only the argument-less forms are recognised here. +// `next`) sitting in operand position, returning its node and true. MRI accepts a +// bare jump as the right operand of `&&`/`||` (`x || return`, `expr && next`) but +// not one carrying a value (`x || return 5` is a syntax error), nor a `retry` +// (`x || retry` is a syntax error), so only these argument-less forms qualify. func (p *Parser) valuelessJumpOperand() (ast.Node, bool) { switch p.cur().Type { case token.RETURN: @@ -1743,9 +1743,6 @@ func (p *Parser) valuelessJumpOperand() (ast.Node, bool) { case token.NEXT: p.advance() return &ast.Next{}, true - case token.RETRY: - p.advance() - return &ast.Retry{}, true } return nil, false } diff --git a/round3_features_test.go b/round3_features_test.go index a576c9b..c591aa7 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -262,7 +262,6 @@ func TestControlFlowInsideParens(t *testing.T) { `(expr if cond)`, `(x if y) || z`, `[1].each { (m.match(s) or next)[1] }`, - `begin; (x or retry); rescue; end`, } { if _, err := parser.Parse(src); err != nil { t.Errorf("Parse(%q): %v", src, err) @@ -281,7 +280,6 @@ func TestControlFlowKeywordAsOperand(t *testing.T) { `a = b || return`, `def f; x or return; end`, `def f; x && return; end`, - `begin; x || retry; rescue; end`, } { if _, err := parser.Parse(src); err != nil { t.Errorf("Parse(%q): %v", src, err) From e7c268a80619daacbb62f9ed047d7ea7e414127e Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 19:01:03 +0200 Subject: [PATCH 09/10] Allow a block's |params| to start on a continued line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The |params| list of a brace/do block may begin on the line(s) after the opener: 'foo {''|x| … }', 'do''|a, b| … end'. parseBlockRest now skips the intervening newlines when the first significant token is a '|' (a body statement can never start with a bare '|', so this is unambiguous), leaving param-less blocks unaffected. Co-Authored-By: Claude Opus 4.8 --- parser.go | 18 ++++++++++++++++++ round3_features_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/parser.go b/parser.go index afb6c84..3ba6b75 100644 --- a/parser.go +++ b/parser.go @@ -196,6 +196,17 @@ func (p *Parser) skipNewlines() { } } +// firstSignificantIs reports whether the first non-NEWLINE token at or after the +// cursor is of type tt, without consuming anything. +func (p *Parser) firstSignificantIs(tt token.Type) bool { + for i := p.pos; i < len(p.toks); i++ { + if p.toks[i].Type != token.NEWLINE { + return p.toks[i].Type == tt + } + } + return false +} + // --- scope --- func (p *Parser) scope() *scope { return p.scopes[len(p.scopes)-1] } @@ -2081,6 +2092,13 @@ func (p *Parser) parseBlockRest(stop map[token.Type]bool, end token.Type, withRe var defaults, prepends []ast.Node var blockParam string splat := -1 + // The `|params|` list may start on the line(s) after the block opener: + // `foo {``|x| … }`, `do``|a, b| … end`. Skip the intervening newlines + // before the leading `|` (a body statement can never begin with a bare `|`, so + // this is unambiguous). The newlines are no-ops for the body either way. + if p.is(token.NEWLINE) && p.firstSignificantIs(token.PIPE) { + p.skipNewlines() + } if p.accept(token.PIPE) { params, defaults, prepends, splat, blockParam = p.parseBlockParams(token.PIPE) p.expect(token.PIPE) diff --git a/round3_features_test.go b/round3_features_test.go index c591aa7..b09b6c4 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -253,6 +253,40 @@ func TestHuggingStringOnReceiverAndConst(t *testing.T) { } } +// --- Feature 9: block params on a continued line --- + +func TestBlockParamsOnContinuedLine(t *testing.T) { + for _, src := range []string{ + "foo {\n|x| x }", + "foo do\n|a, b|\na + b\nend", + "[1].each {\n |x|\n p x\n}", + "foo do\n\n|compare, target, success|\nnil\nend", + } { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + +func TestBlockParamsContinuedShape(t *testing.T) { + call := mustParseSingle(t, "foo do\n|a, b|\na\nend").(*ast.Call) + if call.Block == nil { + t.Fatal("foo do\\n|a,b| …: no block") + } + if len(call.Block.Params) != 2 || call.Block.Params[0] != "a" || call.Block.Params[1] != "b" { + t.Fatalf("block params = %v, want [a b]", call.Block.Params) + } +} + +func TestBlockWithoutParamsStillWorks(t *testing.T) { + // A body that simply starts on the next line must not be read as params. + for _, src := range []string{"foo {\nx }", "foo {\n}", "foo do\nx\nend"} { + if _, err := parser.Parse(src); err != nil { + t.Errorf("Parse(%q): %v", src, err) + } + } +} + // --- Feature 5: control-flow + modifier inside parens --- func TestControlFlowInsideParens(t *testing.T) { From 7a6ff17d60567c84bc61fa6115e42ac5dfaf8c82 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 26 Jun 2026 19:06:16 +0200 Subject: [PATCH 10/10] Restore 100% coverage for the round-3 additions Add parity/edge tests covering every char-literal escape, the percent-delimiter accept/reject branches (alphanumeric, whitespace, '=', multibyte), inline assignment to an existing local and the non-assignable fall-through, and the continued-block-params lookahead; simplify firstSignificantIs to drop a dead branch (the stream always ends in a non-NEWLINE EOF). Co-Authored-By: Claude Opus 4.8 --- lexer/lexer_round3_test.go | 69 ++++++++++++++++++++++++++++++++++++++ parser.go | 12 +++---- round3_features_test.go | 51 ++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 lexer/lexer_round3_test.go diff --git a/lexer/lexer_round3_test.go b/lexer/lexer_round3_test.go new file mode 100644 index 0000000..43a6d42 --- /dev/null +++ b/lexer/lexer_round3_test.go @@ -0,0 +1,69 @@ +package lexer + +import ( + "testing" + + "github.com/go-ruby-parser/parser/token" +) + +// firstType returns the type of the first token of src. +func firstType(src string) token.Type { + return New(src).Tokenize()[0].Type +} + +// TestPercentNonDelimiters checks that a character that cannot delimit a +// percent-literal (an alphanumeric, whitespace, '=', or a multi-byte UTF-8 lead) +// leaves the '%' as the modulo/percent operator rather than opening a literal. +// This exercises every false branch of isPercentDelim. +func TestPercentNonDelimiters(t *testing.T) { + // `%w` followed by a non-delimiter is not a word-array literal. + for _, src := range []string{ + "%wx", // alpha delimiter candidate + "%w9", // digit + "%w ", // space + "%w\t", // tab + } { + if firstType(src) == token.WORDS { + t.Errorf("%q: unexpectedly lexed as a %%w array", src) + } + } + // A bare `%` followed by a non-delimiter is the percent operator. + for _, src := range []string{ + "%=", // compound-assignment, never a literal + "%9", // digit + "%a", // alpha (not q/Q/w/i/r/x/s) + "% ", // space + "%é", // multi-byte UTF-8 lead + "%9x", // digit then alpha + } { + ty := firstType(src) + if ty == token.STRING || ty == token.WORDS || ty == token.REGEXP { + t.Errorf("%q: unexpectedly lexed as a percent literal (%s)", src, ty) + } + } +} + +// TestPercentDelimMultibyteRejected confirms a multibyte lead after a kind letter +// is not accepted as a delimiter (isPercentDelim's >= 0x80 branch). +func TestPercentDelimMultibyteRejected(t *testing.T) { + if firstType("%qé") == token.STRING { + t.Errorf("%%qé: a multibyte delimiter must not open a %%q string") + } +} + +// TestPercentDelimiterAccepted exercises isPercentDelim's success path: assorted +// punctuation delimiters open their literals. +func TestPercentDelimiterAccepted(t *testing.T) { + cases := map[string]token.Type{ + `%w|a b|`: token.WORDS, + `%q!hi!`: token.STRING, + `%r#ab#`: token.REGEXP, + `%i@a b@`: token.SYMBOLS, + `%(hi)`: token.STRING, + } + for src, want := range cases { + if got := firstType(src); got != want { + t.Errorf("%q: first token = %s, want %s", src, got, want) + } + } +} diff --git a/parser.go b/parser.go index 3ba6b75..e165f19 100644 --- a/parser.go +++ b/parser.go @@ -197,14 +197,14 @@ func (p *Parser) skipNewlines() { } // firstSignificantIs reports whether the first non-NEWLINE token at or after the -// cursor is of type tt, without consuming anything. +// cursor is of type tt, without consuming anything. The token stream always ends +// in a (non-NEWLINE) EOF, so the scan always finds a significant token. func (p *Parser) firstSignificantIs(tt token.Type) bool { - for i := p.pos; i < len(p.toks); i++ { - if p.toks[i].Type != token.NEWLINE { - return p.toks[i].Type == tt - } + i := p.pos + for i < len(p.toks) && p.toks[i].Type == token.NEWLINE { + i++ } - return false + return i < len(p.toks) && p.toks[i].Type == tt } // --- scope --- diff --git a/round3_features_test.go b/round3_features_test.go index b09b6c4..29ff4f9 100644 --- a/round3_features_test.go +++ b/round3_features_test.go @@ -60,6 +60,23 @@ func TestCharLiteralBasic(t *testing.T) { } } +func TestCharLiteralAllEscapes(t *testing.T) { + // Exercises every backslash-escape branch of the char literal. + cases := map[string]string{ + "p ?\\r": "\r", + "p ?\\a": "\x07", + "p ?\\b": "\x08", + "p ?\\v": "\x0b", + "p ?\\f": "\x0c", + "p ?\\0": "\x00", + } + for src, want := range cases { + if got := charLitArg(t, src); got != want { + t.Errorf("%q: char value = %q, want %q", src, got, want) + } + } +} + func TestCharLiteralAtExprBegin(t *testing.T) { asn := mustParseSingle(t, `x = ?a`).(*ast.Assign) s, ok := asn.Value.(*ast.StringLit) @@ -287,6 +304,15 @@ func TestBlockWithoutParamsStillWorks(t *testing.T) { } } +// TestBlockOpenerNewlinesToEOF exercises firstSignificantIs's run-to-EOF path: an +// opener followed only by newlines (an unterminated block) still scans for a `|`, +// finds none, and surfaces a clean parse error rather than crashing. +func TestBlockOpenerNewlinesToEOF(t *testing.T) { + if _, err := parser.Parse("foo {\n\n"); err == nil { + t.Fatalf("unterminated block: expected a parse error") + } +} + // --- Feature 5: control-flow + modifier inside parens --- func TestControlFlowInsideParens(t *testing.T) { @@ -369,6 +395,31 @@ func TestInlineAssignmentShape(t *testing.T) { } } +func TestInlineAssignmentToExistingLocal(t *testing.T) { + // When the target is already a known local it parses as a VarRef before the + // `=`, exercising maybeInlineAssign's VarRef branch: `x` is a local, then + // `a && x = 1` assigns to it. + prog := mustParse(t, "x = 0\nif a && x = 1\nend") + ifn := prog.Body[1].(*ast.If) + if _, ok := ifn.Cond.(*ast.BinaryExpr).Right.(*ast.Assign); !ok { + t.Fatalf("a && x = 1 (x local): RHS not an *ast.Assign") + } +} + +func TestInlineAssignmentLeavesNonTargetAlone(t *testing.T) { + // An `=` after a non-assignable operand (a literal, or a call with args) is + // not an inline assignment; the parser surfaces an ordinary error instead of + // silently consuming it (maybeInlineAssign's final return-node path). + for _, src := range []string{ + `if a && 1 = 2; end`, + `if a && foo(x) = 2; end`, + } { + if _, err := parser.Parse(src); err == nil { + t.Errorf("Parse(%q): expected a parse error", src) + } + } +} + func TestInlineAssignmentDoesNotEatComparisons(t *testing.T) { // `==`, `=>`, `=~` must not be swallowed as assignments. for _, src := range []string{`p(a == b)`, `p(a =~ b)`, `x == y`} {