diff --git a/cmd/rbgo/main.go b/cmd/rbgo/main.go index a11efa78..a793356c 100644 --- a/cmd/rbgo/main.go +++ b/cmd/rbgo/main.go @@ -71,7 +71,7 @@ func run(src, name string) error { if err != nil { return err } - iseq, err := compiler.Compile(prog) + iseq, err := compiler.CompileWithEncoding(prog, compiler.MagicSourceEncoding(src)) if err != nil { return err } diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index 07ee8a60..3b37ab19 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -12,6 +12,7 @@ import ( "math/big" "strconv" "strings" + "unicode/utf8" "github.com/go-embedded-ruby/ruby/internal/bytecode" "github.com/go-embedded-ruby/ruby/internal/object" @@ -167,10 +168,23 @@ type Compiler struct { ctxs []*loopCtx // innermost-last stack of break/next targets retryTargets []int // innermost-last stack of begin-body PCs for `retry` stack []*builder + // srcEnc is the source file's encoding (from a `# encoding:` magic comment), + // canonicalised to a registry name, or "" for the UTF-8 default. When set, a + // string literal is tagged with it (see newStrLit) so a literal in a + // `# encoding: binary` file is BINARY, matching MRI. + srcEnc string } -// Compile lowers a Program into the top-level ISeq. +// Compile lowers a Program into the top-level ISeq, treating string literals as +// UTF-8 (the default source encoding). func Compile(prog *ast.Program) (iseq *bytecode.ISeq, err error) { + return CompileWithEncoding(prog, "") +} + +// CompileWithEncoding lowers a Program whose source declared srcEnc (a canonical +// encoding name, or "" for UTF-8) via a `# encoding:` magic comment, so string +// literals carry that encoding. +func CompileWithEncoding(prog *ast.Program, srcEnc string) (iseq *bytecode.ISeq, err error) { defer func() { if r := recover(); r != nil { // Unchecked: a non-compileError is an internal bug and re-panics as a @@ -178,13 +192,87 @@ func Compile(prog *ast.Program) (iseq *bytecode.ISeq, err error) { iseq, err = nil, r.(compileError) } }() - c := &Compiler{} + c := &Compiler{srcEnc: srcEnc} c.push(newBuilder("
", nil)) c.compileBody(prog.Body) c.cur().emit(bytecode.OpReturn, 0, 0) return c.pop().build(), nil } +// newStrLit builds the String constant for a source literal, tagging it with the +// source file's encoding when one was declared. A literal whose bytes are valid +// multi-byte UTF-8 is left UTF-8 regardless: such content comes from a `\u` escape +// (or literal Unicode), which MRI always encodes as UTF-8 even in a non-UTF-8 +// source. Pure-ASCII and byte-oriented (`\x`) content take the source encoding. +func (c *Compiler) newStrLit(s string) object.Value { + if c.srcEnc != "" { + b := []byte(s) + if !(utf8.Valid(b) && !asciiOnlyBytes(b)) { + return object.NewStringBytesEnc(b, c.srcEnc) + } + } + return object.NewString(s) +} + +// asciiOnlyBytes reports whether every byte is 7-bit ASCII. +func asciiOnlyBytes(b []byte) bool { + for _, x := range b { + if x >= 0x80 { + return false + } + } + return true +} + +// MagicSourceEncoding returns the canonical registry encoding name declared by a +// `# encoding:` / `# coding:` magic comment on the first line of src (or the +// second when the first is a shebang), or "" for UTF-8 or no declaration. Only the +// encodings the negotiation cares about are recognised by canonical spelling; +// anything else (including plain UTF-8) yields "" so literals keep the default. +func MagicSourceEncoding(src string) string { + line, rest, _ := strings.Cut(src, "\n") + if strings.HasPrefix(strings.TrimSpace(line), "#!") { + line, _, _ = strings.Cut(rest, "\n") + } + name := magicEncodingName(line) + switch strings.ToLower(name) { + case "binary", "ascii-8bit": + return "ASCII-8BIT" + case "us-ascii", "ascii": + return "US-ASCII" + } + return "" +} + +// magicEncodingName extracts the value of a `coding:`/`encoding:` field from a +// comment line (e.g. `# -*- coding: binary -*-` or `# encoding: binary`), or "". +func magicEncodingName(line string) string { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") { + return "" + } + low := strings.ToLower(line) + i := strings.Index(low, "coding") + if i < 0 { + return "" + } + rest := line[i+len("coding"):] + rest = strings.TrimLeft(rest, ":= \t") + // The value ends at the first whitespace or a `-*-` terminator. + end := len(rest) + for j, r := range rest { + if r == ' ' || r == '\t' || r == ';' { + end = j + break + } + if strings.HasPrefix(rest[j:], "-*-") { + end = j + break + } + } + return strings.TrimSpace(rest[:end]) +} + // CompileWithLocals lowers prog for a Binding eval: it compiles in a child scope // whose parent holds the binding's locals (by slot), so references to them // resolve at depth 1 — reads and writes reach the binding's environment when the @@ -254,7 +342,7 @@ func (c *Compiler) compileNode(n ast.Node) { // Integer/Float, or a Rational for the combined `ri` form). b.emit(bytecode.OpPushConst, b.addConst(&object.Complex{Re: object.IntValue(0), Im: numericValue(v.Value)}), 0) case *ast.StringLit: - b.emit(bytecode.OpPushConst, b.addConst(object.NewString(v.Value)), 0) + b.emit(bytecode.OpPushConst, b.addConst(c.newStrLit(v.Value)), 0) case *ast.StrInterp: // Concatenate each part coerced with to_s onto a growing string. b.emit(bytecode.OpPushConst, b.addConst(object.NewString("")), 0) diff --git a/internal/compiler/magic_encoding_test.go b/internal/compiler/magic_encoding_test.go new file mode 100644 index 00000000..ef015c95 --- /dev/null +++ b/internal/compiler/magic_encoding_test.go @@ -0,0 +1,82 @@ +package compiler + +import ( + "testing" + + "github.com/go-embedded-ruby/ruby/internal/object" + "github.com/go-ruby-parser/parser" +) + +func TestMagicSourceEncoding(t *testing.T) { + cases := []struct{ src, want string }{ + {"# encoding: binary\nx = 1\n", "ASCII-8BIT"}, + {"# encoding: BINARY\n", "ASCII-8BIT"}, + {"# coding: ascii-8bit\n", "ASCII-8BIT"}, + {"# -*- coding: binary -*-\n", "ASCII-8BIT"}, + {"# encoding: us-ascii\n", "US-ASCII"}, + {"# encoding: ascii\n", "US-ASCII"}, + {"# encoding: utf-8\n", ""}, // UTF-8 is the default: no tag + {"# encoding: euc-jp\n", ""}, // unrecognised for tagging: no tag + {"x = 1\n", ""}, // no magic comment + {"puts 1 # encoding: binary\n", ""}, // must be a comment line, not trailing + {"# just a plain comment\n", ""}, // a comment without a coding: field + {"# coding:binary-*-\n", "ASCII-8BIT"}, // value terminated by -*- with no space + {"# coding: binary;\n", "ASCII-8BIT"}, // value terminated by a semicolon + {"#!/usr/bin/env ruby\n# encoding: binary\nx = 1\n", "ASCII-8BIT"}, // 2nd line after shebang + {"", ""}, + } + for _, tc := range cases { + if got := MagicSourceEncoding(tc.src); got != tc.want { + t.Errorf("MagicSourceEncoding(%q) = %q, want %q", tc.src, got, tc.want) + } + } +} + +// firstStringConst returns the encoding name of the first String constant in a +// program compiled with the given source encoding. +func firstStringConst(t *testing.T, src, srcEnc string) string { + t.Helper() + prog, err := parser.Parse(src) + if err != nil { + t.Fatalf("parse: %v", err) + } + iseq, err := CompileWithEncoding(prog, srcEnc) + if err != nil { + t.Fatalf("compile: %v", err) + } + for _, c := range iseq.Consts { + if s, ok := c.(*object.String); ok { + return s.EncName() + } + } + t.Fatalf("no String constant in %q", src) + return "" +} + +func TestCompileWithEncodingTagsLiterals(t *testing.T) { + // A plain ASCII literal in a binary source is tagged BINARY. + if got := firstStringConst(t, `x = "abc"`, "ASCII-8BIT"); got != "ASCII-8BIT" { + t.Errorf(`"abc" in binary source: %q, want "ASCII-8BIT"`, got) + } + // A byte-oriented (\x, high bit) literal in a binary source is tagged BINARY. + if got := firstStringConst(t, `x = "\xff"`, "ASCII-8BIT"); got != "ASCII-8BIT" { + t.Errorf(`"\xff" in binary source: %q, want "ASCII-8BIT"`, got) + } + // A valid multi-byte UTF-8 literal (as a \u escape produces) stays UTF-8 even in + // a binary source, matching MRI's rule that \u forces UTF-8. + if got := firstStringConst(t, `x = "あ"`, "ASCII-8BIT"); got != "UTF-8" { + t.Errorf(`"あ" in binary source: %q, want "UTF-8"`, got) + } + // The default (UTF-8) source leaves every literal untagged / UTF-8. + if got := firstStringConst(t, `x = "abc"`, ""); got != "UTF-8" { + t.Errorf(`"abc" in default source: %q, want "UTF-8"`, got) + } + // Compile (the UTF-8 default entry point) behaves the same as an empty srcEnc. + prog, err := parser.Parse(`x = "abc"`) + if err != nil { + t.Fatalf("parse: %v", err) + } + if _, err := Compile(prog); err != nil { + t.Fatalf("Compile: %v", err) + } +} diff --git a/internal/vm/encoding.go b/internal/vm/encoding.go index fa8d3eff..2a47dcf9 100644 --- a/internal/vm/encoding.go +++ b/internal/vm/encoding.go @@ -96,9 +96,9 @@ func (vm *VM) registerEncoding() { vm.cEncoding.define("name", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return object.NewString(self.(*encodingObj).name) }) - vm.cEncoding.define("to_s", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return object.NewString(self.(*encodingObj).name) - }) + // Encoding#to_s is a true alias of #name (they share one Method record, so + // Encoding.instance_method(:to_s) == Encoding.instance_method(:name), as MRI). + vm.cEncoding.methods["to_s"] = vm.cEncoding.methods["name"] vm.cEncoding.define("inspect", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return object.NewString(self.(*encodingObj).Inspect()) }) @@ -199,37 +199,43 @@ func (vm *VM) registerEncoding() { } // encOperand is one argument to Encoding.compatible?, reduced to what the -// negotiation needs: its governing encoding, its bytes (for String/Symbol content), -// and whether it carries content at all (a String or Symbol) versus being a bare -// Encoding object. +// negotiation needs: its governing encoding, its bytes (for String content), and +// whether it is a real String (whose byte content — its coderange — participates) +// as opposed to a Symbol, Regexp or bare Encoding (whose declared encoding alone +// participates, exactly as MRI treats every non-T_STRING encoding-capable object). type encOperand struct { - enc *encodingObj - bytes []byte - stringLike bool - ok bool + enc *encodingObj + bytes []byte + isStr bool + ok bool } -// is7bit reports whether the operand is ASCII-only *for compatibility purposes*: -// its encoding must be ASCII-compatible AND every byte 7-bit. Content in a -// non-ASCII-compatible encoding (UTF-16, ISO-2022-JP, …) is never 7-bit even when -// empty or all-ASCII, matching MRI's coderange (rb_enc_str_coderange). +// is7bit reports whether a String operand is ASCII-only *for compatibility +// purposes* (MRI's ENC_CODERANGE_7BIT): its encoding must be ASCII-compatible AND +// every byte 7-bit. Content in a non-ASCII-compatible encoding (UTF-16, +// ISO-2022-JP, …) is never 7-bit even when empty or all-ASCII. func (o encOperand) is7bit() bool { return o.enc.asciiCompat && asciiOnly(o.bytes) } -// encOperandOf reduces a compatibility argument to an encOperand: a String keeps -// its tagged encoding and bytes; a Symbol is US-ASCII when ASCII-only else UTF-8; -// an Encoding is itself (no content); anything else has no encoding, making the -// pair incompatible. (Regexp operands are a named residual.) +// encOperandOf reduces a compatibility argument to an encOperand: +// - a String keeps its tagged encoding and bytes (isStr, so its coderange is +// consulted below); +// - a Symbol, Regexp or Encoding contributes only its declared encoding (not a +// String, so its content coderange is never examined) — a Symbol is US-ASCII +// when ASCII-only else UTF-8, a Regexp reports its source encoding, and an +// Encoding is itself; +// - anything else has no encoding, making the pair incompatible. func (vm *VM) encOperandOf(v object.Value) encOperand { switch x := v.(type) { case *object.String: - return encOperand{enc: vm.internEncoding(x.EncName()), bytes: x.Bytes(), stringLike: true, ok: true} + return encOperand{enc: vm.internEncoding(x.EncName()), bytes: x.Bytes(), isStr: true, ok: true} case object.Symbol: - b := []byte(string(x)) name := "UTF-8" - if asciiOnly(b) { + if asciiOnly([]byte(string(x))) { name = "US-ASCII" } - return encOperand{enc: vm.encodings[name], bytes: b, stringLike: true, ok: true} + return encOperand{enc: vm.encodings[name], ok: true} + case *Regexp: + return encOperand{enc: vm.internEncoding(x.encodingName()), ok: true} case *encodingObj: return encOperand{enc: x, ok: true} } @@ -238,65 +244,67 @@ func (vm *VM) encOperandOf(v object.Value) encOperand { // encodingCompatible is Encoding.compatible?(obj1, obj2): the encoding of the // string that would result from concatenating the two, or nil if they cannot be -// combined. It follows MRI's rb_enc_compatible negotiation: -// -// - identical encodings are always compatible; -// - an empty second string yields the first's encoding; -// - against an empty first string, the second's encoding wins unless the first -// is ASCII-compatible and the second is 7-bit; -// - otherwise both encodings must be ASCII-compatible, and the 7-bit (ASCII-only) -// operand adopts the other's encoding; two non-7-bit operands are incompatible. +// combined. It follows MRI's rb_enc_compatible / enc_compatible_latter negotiation +// faithfully, including the String-vs-non-String distinction (only a real String's +// byte coderange participates; a Symbol/Regexp/Encoding contributes its declared +// encoding, with US-ASCII the special "same as contents" case). func (vm *VM) encodingCompatible(a, b object.Value) object.Value { oa := vm.encOperandOf(a) ob := vm.encOperandOf(b) if !oa.ok || !ob.ok { return object.NilV } - e1, e2 := oa.enc, ob.enc - if e1 == e2 { - return e1 + enc1, enc2 := oa.enc, ob.enc + // Identical encodings are always compatible. + if enc1 == enc2 { + return enc1 } - if ob.stringLike && len(ob.bytes) == 0 { - return e1 + // An empty second String takes the first's encoding. + if ob.isStr && len(ob.bytes) == 0 { + return enc1 } - if oa.stringLike && ob.stringLike && len(oa.bytes) == 0 { - if e1.asciiCompat && ob.is7bit() { - return e1 + // An empty first String yields the second's encoding, unless the first is + // ASCII-compatible and the second is 7-bit (then the first's). + if oa.isStr && ob.isStr && len(oa.bytes) == 0 { + if enc1.asciiCompat && ob.is7bit() { + return enc1 } - return e2 + return enc2 } - if !e1.asciiCompat || !e2.asciiCompat { + // From here both encodings must be ASCII-compatible. + if !enc1.asciiCompat || !enc2.asciiCompat { return object.NilV } - if oa.stringLike && ob.stringLike { - // Two content operands: the 7-bit (ASCII-only) one adopts the other's - // encoding; two non-7-bit operands are incompatible. - if ob.is7bit() { - return e1 - } - if oa.is7bit() { - return e2 - } - return object.NilV + // A non-String whose declared encoding is US-ASCII contributes the other + // operand's encoding (MRI's "objects whose encoding is the same of contents"). + if !ob.isStr && enc2.name == "US-ASCII" { + return enc1 } - if oa.stringLike != ob.stringLike { - // One content operand, one bare Encoding. Verified against MRI in both - // argument orders: a US-ASCII Encoding always keeps the string's own - // encoding; otherwise, when the string is ASCII-only, the result is the - // second operand's encoding (the Encoding when it is second, the string's - // encoding when the Encoding is first); a non-ASCII string is incompatible. - str, enc, encSecond := oa, e2, true - if !oa.stringLike { - str, enc, encSecond = ob, e1, false - } - if enc.name == "US-ASCII" { - return str.enc - } - if str.is7bit() { - if encSecond { - return enc + if !oa.isStr && enc1.name == "US-ASCII" { + return enc2 + } + // Orient so s1 is the String (if either is). enc1/enc2 keep referring to the + // original operand positions, as in MRI where only str1/str2 are swapped. + s1, s2 := oa, ob + if !oa.isStr { + s1, s2 = ob, oa + } + if s1.isStr { + cr1 := s1.is7bit() + if s2.isStr { + cr2 := s2.is7bit() + if cr1 != cr2 { + if cr1 { + return enc2 + } + return enc1 } - return str.enc + if cr2 { + return enc1 + } + } + if cr1 { + return enc2 } } return object.NilV diff --git a/internal/vm/encoding_compatible_test.go b/internal/vm/encoding_compatible_test.go new file mode 100644 index 00000000..a78e17d9 --- /dev/null +++ b/internal/vm/encoding_compatible_test.go @@ -0,0 +1,102 @@ +package vm_test + +import "testing" + +// compatName evaluates Encoding.compatible?(a, b) for the given operand source +// expressions and returns the inspected result name (or nil). Every want below is +// verified byte-for-byte against MRI 4.0.6. +func compatName(t *testing.T, a, b string) string { + t.Helper() + src := "r = Encoding.compatible?(" + a + ", " + b + ")\np(r && r.name)\n" + return eval(t, src) +} + +func TestEncodingCompatibleEngine(t *testing.T) { + // Byte-oriented content is built with String#b / #force_encoding so the test + // does not depend on the source file's declared encoding. + cases := []struct { + name, a, b, want string + }{ + // Identical encodings are always compatible (fast path). + {"same_binary", `"abc".b`, `"def".b`, "\"ASCII-8BIT\"\n"}, + {"same_regexp_us_ascii", `/abc/`, `/def/`, "\"US-ASCII\"\n"}, + + // An empty second String takes the first's encoding. + {"empty_second", `"x".dup.force_encoding("UTF-8")`, `""`, "\"UTF-8\"\n"}, + + // An empty first String: first's encoding when first is ASCII-compatible and + // the second is 7-bit; otherwise the second's. + {"empty_first_second_7bit", `"".dup.force_encoding("UTF-8")`, `"\x01\x01".b`, "\"UTF-8\"\n"}, + {"empty_first_second_not7bit", `"".dup.force_encoding("UTF-8")`, `"\x81".b`, "\"ASCII-8BIT\"\n"}, + {"empty_first_not_asciicompat", `"".dup.force_encoding("UTF-16BE")`, `"ab".dup.force_encoding("US-ASCII")`, "\"US-ASCII\"\n"}, + + // A non-ASCII-compatible operand (that is not identical) is incompatible. + {"non_asciicompat_string", `"a".dup.force_encoding("UTF-8")`, `"bb".dup.force_encoding("UTF-16BE")`, "nil\n"}, + + // Two Strings: the 7-bit one adopts the other's encoding; two non-7-bit are + // incompatible. + {"two_strings_second_7bit", `"\x81".b`, `"abc".dup.force_encoding("UTF-8")`, "\"ASCII-8BIT\"\n"}, + {"two_strings_first_7bit", `"abc".dup.force_encoding("UTF-8")`, `"\x81".b`, "\"ASCII-8BIT\"\n"}, + {"two_strings_both_not7bit", `"\x81".b`, `"あ".dup.force_encoding("UTF-8")`, "nil\n"}, + + // Encoding, Encoding operands: US-ASCII is the "same as contents" special. + {"enc_enc_second_us_ascii", `Encoding::UTF_8`, `Encoding::US_ASCII`, "\"UTF-8\"\n"}, + {"enc_enc_first_us_ascii", `Encoding::US_ASCII`, `Encoding::EUC_JP`, "\"EUC-JP\"\n"}, + {"enc_enc_equal", `Encoding::UTF_8`, `Encoding::UTF_8`, "\"UTF-8\"\n"}, + {"enc_enc_differ", `Encoding::UTF_8`, `Encoding::EUC_JP`, "nil\n"}, + {"enc_enc_dummy", `Encoding::UTF_8`, `Encoding::UTF_7`, "nil\n"}, + + // Regexp operands report their source encoding. + {"string_regexp_eucjp", `"hello".dup.force_encoding("utf-8")`, `Regexp.new("\xa4\xa2".dup.force_encoding("euc-jp"))`, "\"EUC-JP\"\n"}, + {"regexp_string_same_eucjp", `Regexp.new("\xa4\xa2".dup.force_encoding("euc-jp"))`, `"hi".dup.force_encoding("euc-jp")`, "\"EUC-JP\"\n"}, + {"regexp_binary_source", `Regexp.new("\xff".b)`, `/abc/`, "\"ASCII-8BIT\"\n"}, + {"regexp_noencoding", `Regexp.new("\xff".b, Regexp::NOENCODING)`, `/abc/`, "\"ASCII-8BIT\"\n"}, + {"regexp_literal_nonascii", `/あ/`, `"x".dup.force_encoding("utf-8")`, "\"UTF-8\"\n"}, + + // Symbol operands: US-ASCII when ASCII-only, else UTF-8. + {"symbol_us_ascii_string_binary", `:abc`, `"\x81".b`, "\"ASCII-8BIT\"\n"}, + {"symbol_nonascii_utf8", `"あ".to_sym`, `"x".dup.force_encoding("utf-8")`, "\"UTF-8\"\n"}, + + // String vs Encoding US-ASCII keeps the String's own encoding. + {"string_binary_enc_us_ascii", `"\x81".b`, `Encoding::US_ASCII`, "\"ASCII-8BIT\"\n"}, + + // Objects without an encoding make the pair incompatible. + {"object_first", `Object.new`, `"x"`, "nil\n"}, + {"object_second", `"x"`, `Object.new`, "nil\n"}, + {"nil_nil", `nil`, `nil`, "nil\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := compatName(t, tc.a, tc.b); got != tc.want { + t.Errorf("Encoding.compatible?(%s, %s) = %q, want %q", tc.a, tc.b, got, tc.want) + } + }) + } +} + +// TestEncodingToSIsAliasOfName checks that Encoding#to_s and #name are one method +// record (a true alias), matching MRI. +func TestEncodingToSIsAliasOfName(t *testing.T) { + if got := eval(t, "p(Encoding.instance_method(:to_s) == Encoding.instance_method(:name))\n"); got != "true\n" { + t.Errorf("to_s alias of name = %q, want %q", got, "true\n") + } + if got := eval(t, `p Encoding::UTF_8.to_s`); got != "\"UTF-8\"\n" { + t.Errorf("Encoding::UTF_8.to_s = %q, want %q", got, "\"UTF-8\"\n") + } +} + +// TestRegexpEncodingFromSource covers Regexp#encoding reporting the source +// String's encoding for a runtime Regexp.new. +func TestRegexpEncodingFromSource(t *testing.T) { + cases := []struct{ src, want string }{ + {`p(/abc/.encoding.name)`, "\"US-ASCII\"\n"}, + {`p(Regexp.new("\xa4\xa2".dup.force_encoding("euc-jp")).encoding.name)`, "\"EUC-JP\"\n"}, + {`p(Regexp.new("\xff".b).encoding.name)`, "\"ASCII-8BIT\"\n"}, + {`p(Regexp.new("あ".dup.force_encoding("utf-8")).encoding.name)`, "\"UTF-8\"\n"}, + } + for _, tc := range cases { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("%s = %q, want %q", tc.src, got, tc.want) + } + } +} diff --git a/internal/vm/frontend_open.go b/internal/vm/frontend_open.go index 0b233776..ba091118 100644 --- a/internal/vm/frontend_open.go +++ b/internal/vm/frontend_open.go @@ -21,5 +21,5 @@ func openParseCompile(src string) (*bytecode.ISeq, error) { if err != nil { return nil, err } - return compiler.Compile(prog) + return compiler.CompileWithEncoding(prog, compiler.MagicSourceEncoding(src)) } diff --git a/internal/vm/regexp.go b/internal/vm/regexp.go index 7349787e..577a6aa2 100644 --- a/internal/vm/regexp.go +++ b/internal/vm/regexp.go @@ -36,6 +36,11 @@ type Regexp struct { // when none was requested. It is reported by #timeout and, when set, applied // to the direct match paths via the engine's WithTimeout. timeout object.Value + // srcEnc is the encoding of the String a runtime Regexp.new was built from, + // which #encoding reports when the source has a non-ASCII byte (e.g. a Regexp + // built from a EUC-JP or BINARY String keeps that encoding). Empty for a source + // literal, whose non-ASCII source is UTF-8 by default. + srcEnc string } // matcher returns the engine Regexp to match with: the receiver's compiled @@ -81,6 +86,9 @@ func (r *Regexp) encodingName() string { if r.noEnc { return "ASCII-8BIT" } + if r.srcEnc != "" { + return r.srcEnc + } return "UTF-8" } @@ -247,6 +255,7 @@ func (vm *VM) regexpNew(args []object.Value) object.Value { } r := vm.compileRegexp(src.Str(), flags).(*Regexp) r.fixedEnc, r.noEnc, r.timeout = fixedEnc, noEnc, timeout + r.srcEnc = src.EncName() return r default: raise("TypeError", "no implicit conversion of %s into String", classNameOf(args[0]))