From af1311fcf655d5d7045e99ac2edbf31c123d7722 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 6 Sep 2026 20:33:03 +0200 Subject: [PATCH] feat(lexer): record regexp encoding flags n/u/e/s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The regexp trailing-flag loop previously collected only i/m/x/o and silently discarded the encoding modifiers n/u/e/s, which blocked rbgo regexp encoding conformance. MRI records an ARG_ENCODING for each of these (rb_char_to_option_kcode in re.c): n=ASCII-8BIT, u=UTF-8, e=EUC-JP, s=Windows-31J. Append them to Flags in source order for both the /…/ (lexRegexp) and %r{…} (lexPercentRXS) forms; any other trailing lowercase letter is still consumed harmlessly. Update the Flags doc comments on token.Token and ast.RegexpLit, and replace the old TestRegexpEncodingFlagsStillDropped with TestRegexpEncodingFlagsRecorded (plus a %r variant). Co-Authored-By: Claude Opus 4.8 --- ast/ast.go | 5 +++-- lexer/lexer.go | 11 +++++++---- percent_rxs_test.go | 38 ++++++++++++++++++++++++++++++++------ token/token.go | 2 +- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ast/ast.go b/ast/ast.go index d85b3bc..abcb267 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -36,8 +36,9 @@ type StringLit struct{ Value string } // SymbolLit is a symbol literal (:name); Name excludes the leading colon. type SymbolLit struct{ Name string } -// RegexpLit is a regexp literal /source/flags. Flags holds the subset of the -// flag letters i, m, x that were present. +// RegexpLit is a regexp literal /source/flags. Flags holds, in source order, +// the subset of the flag letters i, m, x, o and the encoding flags n (ASCII-8BIT), +// u (UTF-8), e (EUC-JP) and s (Windows-31J) that were present. type RegexpLit struct { Source string Flags string diff --git a/lexer/lexer.go b/lexer/lexer.go index 1539221..dd7ac07 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -1025,8 +1025,9 @@ func (l *Lexer) lexIvar(spaceBefore bool, line, col int) token.Token { // cursor. Escapes are preserved verbatim into the source (so \d, \. and the // like reach the engine untouched) except that an escaped delimiter \/ becomes // a literal '/'. Trailing flag letters i, m, x and o (the "interpolate once" -// flag) are collected into Flags; any other trailing letters (e.g. the encoding -// flags n/u/e/s) are ignored gracefully (consumed but not recorded). +// flag) plus the encoding flags n (ASCII-8BIT), u (UTF-8), e (EUC-JP) and s +// (Windows-31J) are collected into Flags in source order; any other trailing +// lowercase letters are consumed but not recorded (ignored gracefully). func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token { l.advance() // opening '/' var src []byte @@ -1071,7 +1072,8 @@ func (l *Lexer) lexRegexp(spaceBefore bool, line, col int) token.Token { break } l.advance() - if c == 'i' || c == 'm' || c == 'x' || c == 'o' { + if c == 'i' || c == 'm' || c == 'x' || c == 'o' || + c == 'n' || c == 'u' || c == 'e' || c == 's' { flags = append(flags, c) } } @@ -1563,7 +1565,8 @@ func (l *Lexer) lexPercentRXS(spaceBefore bool, line, col int) token.Token { break } l.advance() - if c == 'i' || c == 'm' || c == 'x' || c == 'o' { + if c == 'i' || c == 'm' || c == 'x' || c == 'o' || + c == 'n' || c == 'u' || c == 'e' || c == 's' { flags = append(flags, c) } } diff --git a/percent_rxs_test.go b/percent_rxs_test.go index 150a19d..06fc043 100644 --- a/percent_rxs_test.go +++ b/percent_rxs_test.go @@ -65,12 +65,38 @@ func TestPercentRegexpOnceFlag(t *testing.T) { } } -func TestRegexpEncodingFlagsStillDropped(t *testing.T) { - // The encoding flags n/u/e/s remain consumed-but-unrecorded (as before), so a - // bare /x/n exposes no flags — only i/m/x/o are meaningful to the VM. - rx := parseOne(t, "/x/n").(*ast.RegexpLit) - if rx.Flags != "" { - t.Errorf("Flags = %q, want empty (encoding flag n dropped)", rx.Flags) +func TestRegexpEncodingFlagsRecorded(t *testing.T) { + // The encoding flags n/u/e/s are now recorded (matching MRI, which records an + // ARG_ENCODING for each): n=ASCII-8BIT, u=UTF-8, e=EUC-JP, s=Windows-31J. They + // are collected into Flags in source order alongside i/m/x/o. + cases := []struct{ src, want string }{ + {"/x/n", "n"}, // n -> ASCII-8BIT (BINARY) + {"/x/u", "u"}, // u -> UTF-8 + {"/x/e", "e"}, // e -> EUC-JP + {"/x/s", "s"}, // s -> Windows-31J (SJIS) + {"/a/mn", "mn"}, // encoding flag recorded after i/m/x/o, in source order + {"/a/un", "un"}, // multiple trailing letters preserved in the order written + } + for _, c := range cases { + rx := parseOne(t, c.src).(*ast.RegexpLit) + if rx.Flags != c.want { + t.Errorf("%s: Flags = %q, want %q", c.src, rx.Flags, c.want) + } + } +} + +func TestPercentRegexpEncodingFlagsRecorded(t *testing.T) { + // The %r{…} form records the same encoding flags as the /…/ form. + cases := []struct{ src, want string }{ + {"%r{x}n", "n"}, + {"%r{x}u", "u"}, + {"%r{x}mn", "mn"}, + } + for _, c := range cases { + rx := parseOne(t, c.src).(*ast.RegexpLit) + if rx.Flags != c.want { + t.Errorf("%s: Flags = %q, want %q", c.src, rx.Flags, c.want) + } } } diff --git a/token/token.go b/token/token.go index d7d26a2..f7408e6 100644 --- a/token/token.go +++ b/token/token.go @@ -155,7 +155,7 @@ var Keywords = map[string]Type{ type Token struct { Type Type Lit string - Flags string // regexp flag letters (i, m, x), only set for REGEXP tokens + Flags string // regexp flag letters in source order (i, m, x, o and the encoding flags n, u, e, s), only set for REGEXP tokens Line int Col int SpaceBefore bool // whitespace immediately preceded this token (MRI spaceSeen)