diff --git a/internal/vm/builtins.go b/internal/vm/builtins.go index 7cc6839d..8395e8f2 100644 --- a/internal/vm/builtins.go +++ b/internal/vm/builtins.go @@ -2057,6 +2057,17 @@ func (vm *VM) bootstrap() { strConcatFn := func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { s := self.(*object.String) vm.checkFrozen(s) + // Freeze any argument that aliases the receiver before the loop mutates it, + // so `a.concat(a, a)` contributes a's INITIAL value at every position (MRI: + // "hellohellohello", not "hellohellohellohello"). + if len(args) > 1 { + args = append([]object.Value(nil), args...) + for i, a := range args { + if str, ok := a.(*object.String); ok && str == s { + args[i] = str.Dup() + } + } + } for _, a := range args { if u, ok := a.(object.KeyUnwrapper); ok { // a String subclass instance if w, wrapped := u.HashUnwrap(); wrapped { @@ -5224,7 +5235,23 @@ func chrEncode(n int64, enc string) []byte { raise("RangeError", "%d out of char range", n) } return []byte{byte(n)} - default: // ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte + default: + // A legacy multibyte encoding (EUC-JP, Shift_JIS, …) interprets the Integer + // as a big-endian byte sequence that must form exactly one valid character + // (MRI's rb_enc_mbcput + precise-mbclen check); a lone lead byte such as + // 0x81 in EUC-JP is in byte range yet not a character, so it raises + // "invalid codepoint" rather than the plain "out of char range". + if codec, isMB := xtextEncodings[enc]; isMB { + if n >= 0 { + be := bigEndianBytes(uint64(n)) + if out, err := codec.NewDecoder().Bytes(be); err == nil && + utf8.RuneCount(out) == 1 && !strings.ContainsRune(string(out), utf8.RuneError) { + return be + } + } + raise("RangeError", "invalid codepoint 0x%x in %s", n, enc) + } + // ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte. if n < 0 || n > 255 { raise("RangeError", "%d out of char range", n) } @@ -5232,6 +5259,24 @@ func chrEncode(n int64, enc string) []byte { } } +// bigEndianBytes returns n as its minimal big-endian byte sequence (a single +// zero byte for 0). It is how an Integer codepoint is unpacked into the encoded +// bytes for a legacy multibyte encoding. +func bigEndianBytes(n uint64) []byte { + if n == 0 { + return []byte{0} + } + var b []byte + for n > 0 { + b = append(b, byte(n)) + n >>= 8 + } + for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 { + b[i], b[j] = b[j], b[i] + } + return b +} + // codepointAppend encodes the codepoint n for `String#<<`/#concat with an Integer // argument, returning its bytes and the resulting string encoding. It mirrors // chrEncode but, like MRI, promotes a US-ASCII receiver to ASCII-8BIT when the diff --git a/internal/vm/converter_impl.go b/internal/vm/converter_impl.go index 66fe69bb..f5787d87 100644 --- a/internal/vm/converter_impl.go +++ b/internal/vm/converter_impl.go @@ -37,14 +37,16 @@ func (vm *VM) converterNew(args []object.Value) object.Value { if len(args) < 2 { raise("ArgumentError", "wrong number of arguments (given %d, expected 2..3)", len(args)) } - c := &converterObj{ - src: vm.encodingArg(args[0]).name, - dst: vm.encodingArg(args[1]).name, - status: "source_buffer_empty", - } - if c.src == c.dst { - raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", c.src, c.dst) - } + // MRI resolves both endpoints through rb_econv_open: an unknown name, an + // identical src/dst, or a pair with no available codec all raise + // ConverterNotFoundError (never ArgumentError). An unresolvable name is kept + // verbatim in the message; a resolvable one is canonicalised. + src, srcOK := vm.converterEncName(args[0]) + dst, dstOK := vm.converterEncName(args[1]) + if !srcOK || !dstOK || src == dst || !vm.converterDeliverable(src, dst) { + raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", src, dst) + } + c := &converterObj{src: src, dst: dst, status: "source_buffer_empty"} c.repl, c.replEnc = defaultReplacementBytes(c.dst) if len(args) >= 3 { @@ -53,6 +55,46 @@ func (vm *VM) converterNew(args []object.Value) object.Value { return c } +// converterEncName resolves one Converter.new endpoint to a canonical encoding +// name. An Encoding, or a String / #to_str naming a registered encoding, yields +// (canonicalName, true); an unregistered name yields the raw string with +// ok=false, so the caller raises ConverterNotFoundError keeping the name exactly +// as given (as MRI does). A non-String, non-Encoding raises TypeError. +func (vm *VM) converterEncName(v object.Value) (string, bool) { + switch e := v.(type) { + case *encodingObj: + return e.name, true + case *object.String: + if enc, ok := vm.findEncoding(e.Str()); ok { + return enc.name, true + } + return e.Str(), false + } + if vm.respondsToDynamic(v, "to_str") { + if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok { + if enc, ok := vm.findEncoding(s.Str()); ok { + return enc.name, true + } + return s.Str(), false + } + } + raise("TypeError", "no implicit conversion of %s into String", classNameOf(v)) + return "", false +} + +// converterDeliverable reports whether rbgo can build every hop of a src→dst +// conversion (both names already canonical and distinct). It mirrors the hop +// check buildConvpath performs, without allocating the path Array. +func (vm *VM) converterDeliverable(src, dst string) bool { + path := pathList(src, dst) + for i := 0; i+1 < len(path); i++ { + if !vm.hopConvertible(path[i], path[i+1]) { + return false + } + } + return true +} + // applyConverterOpts reads Converter.new's third argument: an options Hash (its // keys drive the flags), or an Integer whose INVALID_REPLACE / UNDEF_REPLACE / // CRLF_NEWLINE_DECORATOR bits set the same flags. diff --git a/internal/vm/converter_test.go b/internal/vm/converter_test.go index fc5b1066..a8ede6b2 100644 --- a/internal/vm/converter_test.go +++ b/internal/vm/converter_test.go @@ -139,8 +139,9 @@ func TestConverterConvert(t *testing.T) { {`c = Encoding::Converter.new("utf-8","iso-8859-1"); c.finish; c.convert("a")`, "convert after finish"}, // A trailing incomplete sequence buffers, then #finish raises. {`c = Encoding::Converter.new("euc-jp","iso-8859-1"); c.convert("\xa4".b); c.finish`, "incomplete"}, - // Destination without a codec: even ASCII is undefined. - {`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "UndefinedConversionError"}, + // Destination without a codec: MRI rejects the pair at construction time + // (rb_econv_open) rather than at convert time. + {`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "ConverterNotFoundError"}, {`Encoding::Converter.new("utf-8","shift_jis").convert("😀")`, "UndefinedConversionError"}, } for _, c := range errCases { @@ -210,7 +211,6 @@ func TestConverterPrimitive(t *testing.T) { // x/text source fallback: a bare invalid high byte and a truncated lead. {`c = Encoding::Converter.new("shift_jis","utf-8"); s = "\xfd\xfd\xfd\xfd".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"}, {`c = Encoding::Converter.new("shift_jis","utf-8"); p c.primitive_convert("\x82".b, String.new, nil, nil, partial_input: true)`, ":source_buffer_empty\n"}, - {`c = Encoding::Converter.new("UTF-7","utf-8"); s = "\x80".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"}, } for _, c := range cases { if got := eval(t, c.src); got != c.want { diff --git a/internal/vm/encoding.go b/internal/vm/encoding.go index 2a47dcf9..cf3c6033 100644 --- a/internal/vm/encoding.go +++ b/internal/vm/encoding.go @@ -384,8 +384,12 @@ func (vm *VM) registerStringEncoding() { d.Enc = "ASCII-8BIT" return d }) - vm.cString.define("ascii_only?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return object.Bool(asciiOnly(self.(*object.String).Bytes())) + vm.cString.define("ascii_only?", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + // #ascii_only? is ENC_CODERANGE_7BIT: content in a non-ASCII-compatible + // encoding (UTF-16/32, …) is never 7-bit, even when empty or all-ASCII in + // bytes, so it reports false regardless of the bytes. + s := self.(*object.String) + return object.Bool(vm.internEncoding(s.EncName()).asciiCompat && asciiOnly(s.Bytes())) }) vm.cString.define("valid_encoding?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { s := self.(*object.String) diff --git a/internal/vm/encoding_compat_test.go b/internal/vm/encoding_compat_test.go new file mode 100644 index 00000000..add71579 --- /dev/null +++ b/internal/vm/encoding_compat_test.go @@ -0,0 +1,162 @@ +package vm_test + +import ( + "strings" + "testing" +) + +// TestConverterNewNotFound covers Encoding::Converter.new's endpoint resolution. +// MRI's rb_econv_open raises Encoding::ConverterNotFoundError — never +// ArgumentError — for an unknown name, an identical src/dst, or a known but +// undeliverable pair (any pair through the dummy UTF-7). A non-String, +// non-Encoding argument raises TypeError. Every expectation is verified against +// MRI 4.0.x. +func TestConverterNewNotFound(t *testing.T) { + cases := []struct { + name, src, wantSub string + }{ + // Unknown source keeps its name verbatim; the known destination is + // canonicalised ("utf-8" -> "UTF-8"). + {"unknown_src", `Encoding::Converter.new("foo-bar", "utf-8")`, + "code converter not found (foo-bar to UTF-8)"}, + {"unknown_dst", `Encoding::Converter.new("utf-8", "foo-bar")`, + "code converter not found (UTF-8 to foo-bar)"}, + {"both_unknown", `Encoding::Converter.new("foo", "bar")`, + "code converter not found (foo to bar)"}, + // Known but undeliverable: the dummy UTF-7 has no codec in either direction. + {"undeliverable_to_utf7", `Encoding::Converter.new("utf-8", "utf-7")`, + "code converter not found (UTF-8 to UTF-7)"}, + {"undeliverable_from_utf7", `Encoding::Converter.new("utf-7", "utf-8")`, + "code converter not found (UTF-7 to UTF-8)"}, + // Identical endpoints are their own not-found case. + {"identical", `Encoding::Converter.new("utf-8", "utf-8")`, + "code converter not found (UTF-8 to UTF-8)"}, + // An Encoding object resolves like its canonical name. + {"encoding_object", `Encoding::Converter.new(Encoding::UTF_8, Encoding::UTF_7)`, + "code converter not found (UTF-8 to UTF-7)"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := runErr(t, c.src) + if err == nil { + t.Fatalf("src=%q: expected error, got none", c.src) + } + if !strings.Contains(err.Error(), "ConverterNotFoundError") { + t.Errorf("src=%q: want ConverterNotFoundError, got %v", c.src, err) + } + if !strings.Contains(err.Error(), c.wantSub) { + t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err) + } + }) + } +} + +func TestConverterNewTypeError(t *testing.T) { + for _, src := range []string{ + `Encoding::Converter.new(123, "utf-8")`, + `Encoding::Converter.new("utf-8", nil)`, + } { + err := runErr(t, src) + if err == nil || !strings.Contains(err.Error(), "TypeError") { + t.Errorf("src=%q: want TypeError, got %v", src, err) + } + } +} + +// TestConverterNewValidPair confirms a genuinely deliverable pair still +// constructs (the deliverability check must not over-reject). The final case +// resolves an endpoint through #to_str. +func TestConverterNewValidPair(t *testing.T) { + for _, src := range []string{ + `p Encoding::Converter.new("utf-8", "utf-16le").class.name`, + `p Encoding::Converter.new("euc-jp", "utf-8").class.name`, + `p Encoding::Converter.new(Encoding::UTF_8, Encoding::SHIFT_JIS).class.name`, + `o = Object.new; def o.to_str; "utf-16le"; end; p Encoding::Converter.new(o, "utf-8").class.name`, + } { + if got := eval(t, src); got != "\"Encoding::Converter\"\n" { + t.Errorf("src=%q: got %q", src, got) + } + } +} + +// TestConverterNewToStrUnknown resolves an endpoint through #to_str to an +// unregistered name: the raw #to_str result is kept in the not-found message. +func TestConverterNewToStrUnknown(t *testing.T) { + src := `o = Object.new; def o.to_str; "bogus-enc"; end; Encoding::Converter.new(o, "utf-8")` + err := runErr(t, src) + if err == nil || !strings.Contains(err.Error(), "ConverterNotFoundError") || + !strings.Contains(err.Error(), "code converter not found (bogus-enc to UTF-8)") { + t.Errorf("src=%q: got %v", src, err) + } +} + +// TestConcatAliasedReceiver covers the snapshot fix for a receiver that also +// appears among its own concat arguments: MRI contributes the receiver's INITIAL +// value at each position. +func TestConcatAliasedReceiver(t *testing.T) { + cases := []struct{ src, want string }{ + {`a = "hello"; a.concat(a, a); p a`, "\"hellohellohello\"\n"}, + {`a = "ab"; a.concat(a); p a`, "\"abab\"\n"}, // single arg, one append + {`a = "x"; a.concat("y", a, "z"); p a`, "\"xyxz\"\n"}, // mixed, self in the middle + {`p "ab".concat("c", "d")`, "\"abcd\"\n"}, // no aliasing, multi-arg + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } +} + +// TestAsciiOnlyEncoding covers that #ascii_only? is the 7-bit coderange, not a +// bare byte scan: content in a non-ASCII-compatible encoding is never 7-bit, +// even when empty or all-ASCII in bytes. +func TestAsciiOnlyEncoding(t *testing.T) { + cases := []struct{ src, want string }{ + {`p "abc".ascii_only?`, "true\n"}, + {`p "".ascii_only?`, "true\n"}, // empty UTF-8 is 7-bit + {`p "".force_encoding("UTF-16LE").ascii_only?`, "false\n"}, + {`p "abc".force_encoding("UTF-16BE").ascii_only?`, "false\n"}, + {`p "\xe3\x81\x82".ascii_only?`, "false\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } +} + +// TestConcatMultibyteCodepoint covers appending an Integer codepoint to a string +// in a legacy multibyte encoding: a valid codepoint is unpacked big-endian, an +// in-range but non-character byte (a lone lead byte) raises RangeError, and +// single-byte encodings still accept any 0..255 byte. +func TestConcatMultibyteCodepoint(t *testing.T) { + ok := []struct{ src, want string }{ + {`s = "".force_encoding("euc-jp"); s.concat(0xa4a2); p s.bytes`, "[164, 162]\n"}, + {`s = "".force_encoding("shift_jis"); s.concat(0x82a0); p s.bytes`, "[130, 160]\n"}, + {`p 0xa4a2.chr("euc-jp").bytes`, "[164, 162]\n"}, + {`p 0.chr("euc-jp").bytes`, "[0]\n"}, // codepoint 0 unpacks to a single NUL byte + {`s = "".force_encoding("iso-8859-1"); s.concat(0x81); p s.bytes`, "[129]\n"}, + } + for _, c := range ok { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } + bad := []struct{ src, wantSub string }{ + {`"".force_encoding("euc-jp").concat(0x81)`, "invalid codepoint 0x81 in EUC-JP"}, + {`"".force_encoding("shift_jis").concat(0x81)`, "invalid codepoint 0x81 in Shift_JIS"}, + {`0x81.chr("euc-jp")`, "invalid codepoint 0x81 in EUC-JP"}, + {`"".force_encoding("euc-jp").concat(-1)`, "invalid codepoint"}, + {`"".force_encoding("us-ascii").concat(256)`, "out of char range"}, + } + for _, c := range bad { + err := runErr(t, c.src) + if err == nil || !strings.Contains(err.Error(), "RangeError") { + t.Errorf("src=%q: want RangeError, got %v", c.src, err) + continue + } + if !strings.Contains(err.Error(), c.wantSub) { + t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err) + } + } +}