From 42bce667f664bf25ba8d3b64b0719a67777af987 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 31 Aug 2026 15:38:50 +0200 Subject: [PATCH 1/4] vm: Converter.new raises ConverterNotFoundError; fix aliased String#concat Encoding::Converter.new now matches MRI's rb_econv_open: an unknown encoding name, or a known but undeliverable pair (e.g. any pair involving the dummy UTF-7), raises Encoding::ConverterNotFoundError with a "code converter not found (SRC to DST)" message (unresolvable names kept verbatim, resolvable ones canonicalised) instead of ArgumentError or silently constructing. A non-String, non-Encoding argument still raises TypeError. String#concat/<< with an argument that aliases the receiver (a.concat(a, a)) now contributes a's INITIAL value at every position ("hellohellohello"), by freezing aliased arguments before the append loop mutates the receiver. Two converter_test cases that asserted the old (non-MRI) behaviour are corrected to expect ConverterNotFoundError at construction. Co-Authored-By: Claude Opus 4.8 --- internal/vm/builtins.go | 11 +++++++ internal/vm/converter_impl.go | 58 ++++++++++++++++++++++++++++++----- internal/vm/converter_test.go | 6 ++-- 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/internal/vm/builtins.go b/internal/vm/builtins.go index 7cc6839d..cafb8b60 100644 --- a/internal/vm/builtins.go +++ b/internal/vm/builtins.go @@ -2057,6 +2057,17 @@ func (vm *VM) bootstrap() { strConcatFn := func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { s := self.(*object.String) vm.checkFrozen(s) + // Freeze any argument that aliases the receiver before the loop mutates it, + // so `a.concat(a, a)` contributes a's INITIAL value at every position (MRI: + // "hellohellohello", not "hellohellohellohello"). + if len(args) > 1 { + args = append([]object.Value(nil), args...) + for i, a := range args { + if str, ok := a.(*object.String); ok && str == s { + args[i] = str.Dup() + } + } + } for _, a := range args { if u, ok := a.(object.KeyUnwrapper); ok { // a String subclass instance if w, wrapped := u.HashUnwrap(); wrapped { diff --git a/internal/vm/converter_impl.go b/internal/vm/converter_impl.go index 66fe69bb..f5787d87 100644 --- a/internal/vm/converter_impl.go +++ b/internal/vm/converter_impl.go @@ -37,14 +37,16 @@ func (vm *VM) converterNew(args []object.Value) object.Value { if len(args) < 2 { raise("ArgumentError", "wrong number of arguments (given %d, expected 2..3)", len(args)) } - c := &converterObj{ - src: vm.encodingArg(args[0]).name, - dst: vm.encodingArg(args[1]).name, - status: "source_buffer_empty", - } - if c.src == c.dst { - raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", c.src, c.dst) - } + // MRI resolves both endpoints through rb_econv_open: an unknown name, an + // identical src/dst, or a pair with no available codec all raise + // ConverterNotFoundError (never ArgumentError). An unresolvable name is kept + // verbatim in the message; a resolvable one is canonicalised. + src, srcOK := vm.converterEncName(args[0]) + dst, dstOK := vm.converterEncName(args[1]) + if !srcOK || !dstOK || src == dst || !vm.converterDeliverable(src, dst) { + raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", src, dst) + } + c := &converterObj{src: src, dst: dst, status: "source_buffer_empty"} c.repl, c.replEnc = defaultReplacementBytes(c.dst) if len(args) >= 3 { @@ -53,6 +55,46 @@ func (vm *VM) converterNew(args []object.Value) object.Value { return c } +// converterEncName resolves one Converter.new endpoint to a canonical encoding +// name. An Encoding, or a String / #to_str naming a registered encoding, yields +// (canonicalName, true); an unregistered name yields the raw string with +// ok=false, so the caller raises ConverterNotFoundError keeping the name exactly +// as given (as MRI does). A non-String, non-Encoding raises TypeError. +func (vm *VM) converterEncName(v object.Value) (string, bool) { + switch e := v.(type) { + case *encodingObj: + return e.name, true + case *object.String: + if enc, ok := vm.findEncoding(e.Str()); ok { + return enc.name, true + } + return e.Str(), false + } + if vm.respondsToDynamic(v, "to_str") { + if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok { + if enc, ok := vm.findEncoding(s.Str()); ok { + return enc.name, true + } + return s.Str(), false + } + } + raise("TypeError", "no implicit conversion of %s into String", classNameOf(v)) + return "", false +} + +// converterDeliverable reports whether rbgo can build every hop of a src→dst +// conversion (both names already canonical and distinct). It mirrors the hop +// check buildConvpath performs, without allocating the path Array. +func (vm *VM) converterDeliverable(src, dst string) bool { + path := pathList(src, dst) + for i := 0; i+1 < len(path); i++ { + if !vm.hopConvertible(path[i], path[i+1]) { + return false + } + } + return true +} + // applyConverterOpts reads Converter.new's third argument: an options Hash (its // keys drive the flags), or an Integer whose INVALID_REPLACE / UNDEF_REPLACE / // CRLF_NEWLINE_DECORATOR bits set the same flags. diff --git a/internal/vm/converter_test.go b/internal/vm/converter_test.go index fc5b1066..a8ede6b2 100644 --- a/internal/vm/converter_test.go +++ b/internal/vm/converter_test.go @@ -139,8 +139,9 @@ func TestConverterConvert(t *testing.T) { {`c = Encoding::Converter.new("utf-8","iso-8859-1"); c.finish; c.convert("a")`, "convert after finish"}, // A trailing incomplete sequence buffers, then #finish raises. {`c = Encoding::Converter.new("euc-jp","iso-8859-1"); c.convert("\xa4".b); c.finish`, "incomplete"}, - // Destination without a codec: even ASCII is undefined. - {`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "UndefinedConversionError"}, + // Destination without a codec: MRI rejects the pair at construction time + // (rb_econv_open) rather than at convert time. + {`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "ConverterNotFoundError"}, {`Encoding::Converter.new("utf-8","shift_jis").convert("😀")`, "UndefinedConversionError"}, } for _, c := range errCases { @@ -210,7 +211,6 @@ func TestConverterPrimitive(t *testing.T) { // x/text source fallback: a bare invalid high byte and a truncated lead. {`c = Encoding::Converter.new("shift_jis","utf-8"); s = "\xfd\xfd\xfd\xfd".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"}, {`c = Encoding::Converter.new("shift_jis","utf-8"); p c.primitive_convert("\x82".b, String.new, nil, nil, partial_input: true)`, ":source_buffer_empty\n"}, - {`c = Encoding::Converter.new("UTF-7","utf-8"); s = "\x80".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"}, } for _, c := range cases { if got := eval(t, c.src); got != c.want { From 5dad4241c05e68ba92dd7e7cd6c9ff6909f58854 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 31 Aug 2026 15:48:01 +0200 Subject: [PATCH 2/4] vm: validate Integer codepoint against a multibyte encoding on concat/chr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String#concat/<< and Integer#chr with an Integer argument now interpret the value as a big-endian byte sequence that must form exactly one valid character in a legacy multibyte encoding (EUC-JP, Shift_JIS, …), matching MRI's rb_enc_mbcput + precise-mbclen check: - a valid multibyte codepoint (e.g. 0xa4a2 in EUC-JP) is now accepted and unpacked to its bytes, where before rbgo wrongly raised "out of char range"; - a lone lead byte such as 0x81 in EUC-JP/Shift_JIS is in byte range but is not a character, so it now raises "invalid codepoint 0x81 in EUC-JP" instead of silently appending the byte. Single-byte encodings (ISO-8859-1, BINARY, …) keep the plain 0..255 rule. New encoding_compat_test.go (package vm_test) covers Converter.new not-found / type-error / valid-pair paths, the aliased-receiver concat snapshot, and the multibyte codepoint validation. Co-Authored-By: Claude Opus 4.8 --- internal/vm/builtins.go | 36 +++++++- internal/vm/encoding_compat_test.go | 130 ++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 internal/vm/encoding_compat_test.go diff --git a/internal/vm/builtins.go b/internal/vm/builtins.go index cafb8b60..8395e8f2 100644 --- a/internal/vm/builtins.go +++ b/internal/vm/builtins.go @@ -5235,7 +5235,23 @@ func chrEncode(n int64, enc string) []byte { raise("RangeError", "%d out of char range", n) } return []byte{byte(n)} - default: // ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte + default: + // A legacy multibyte encoding (EUC-JP, Shift_JIS, …) interprets the Integer + // as a big-endian byte sequence that must form exactly one valid character + // (MRI's rb_enc_mbcput + precise-mbclen check); a lone lead byte such as + // 0x81 in EUC-JP is in byte range yet not a character, so it raises + // "invalid codepoint" rather than the plain "out of char range". + if codec, isMB := xtextEncodings[enc]; isMB { + if n >= 0 { + be := bigEndianBytes(uint64(n)) + if out, err := codec.NewDecoder().Bytes(be); err == nil && + utf8.RuneCount(out) == 1 && !strings.ContainsRune(string(out), utf8.RuneError) { + return be + } + } + raise("RangeError", "invalid codepoint 0x%x in %s", n, enc) + } + // ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte. if n < 0 || n > 255 { raise("RangeError", "%d out of char range", n) } @@ -5243,6 +5259,24 @@ func chrEncode(n int64, enc string) []byte { } } +// bigEndianBytes returns n as its minimal big-endian byte sequence (a single +// zero byte for 0). It is how an Integer codepoint is unpacked into the encoded +// bytes for a legacy multibyte encoding. +func bigEndianBytes(n uint64) []byte { + if n == 0 { + return []byte{0} + } + var b []byte + for n > 0 { + b = append(b, byte(n)) + n >>= 8 + } + for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 { + b[i], b[j] = b[j], b[i] + } + return b +} + // codepointAppend encodes the codepoint n for `String#<<`/#concat with an Integer // argument, returning its bytes and the resulting string encoding. It mirrors // chrEncode but, like MRI, promotes a US-ASCII receiver to ASCII-8BIT when the diff --git a/internal/vm/encoding_compat_test.go b/internal/vm/encoding_compat_test.go new file mode 100644 index 00000000..590d74ec --- /dev/null +++ b/internal/vm/encoding_compat_test.go @@ -0,0 +1,130 @@ +package vm_test + +import ( + "strings" + "testing" +) + +// TestConverterNewNotFound covers Encoding::Converter.new's endpoint resolution. +// MRI's rb_econv_open raises Encoding::ConverterNotFoundError — never +// ArgumentError — for an unknown name, an identical src/dst, or a known but +// undeliverable pair (any pair through the dummy UTF-7). A non-String, +// non-Encoding argument raises TypeError. Every expectation is verified against +// MRI 4.0.x. +func TestConverterNewNotFound(t *testing.T) { + cases := []struct { + name, src, wantSub string + }{ + // Unknown source keeps its name verbatim; the known destination is + // canonicalised ("utf-8" -> "UTF-8"). + {"unknown_src", `Encoding::Converter.new("foo-bar", "utf-8")`, + "code converter not found (foo-bar to UTF-8)"}, + {"unknown_dst", `Encoding::Converter.new("utf-8", "foo-bar")`, + "code converter not found (UTF-8 to foo-bar)"}, + {"both_unknown", `Encoding::Converter.new("foo", "bar")`, + "code converter not found (foo to bar)"}, + // Known but undeliverable: the dummy UTF-7 has no codec in either direction. + {"undeliverable_to_utf7", `Encoding::Converter.new("utf-8", "utf-7")`, + "code converter not found (UTF-8 to UTF-7)"}, + {"undeliverable_from_utf7", `Encoding::Converter.new("utf-7", "utf-8")`, + "code converter not found (UTF-7 to UTF-8)"}, + // Identical endpoints are their own not-found case. + {"identical", `Encoding::Converter.new("utf-8", "utf-8")`, + "code converter not found (UTF-8 to UTF-8)"}, + // An Encoding object resolves like its canonical name. + {"encoding_object", `Encoding::Converter.new(Encoding::UTF_8, Encoding::UTF_7)`, + "code converter not found (UTF-8 to UTF-7)"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := runErr(t, c.src) + if err == nil { + t.Fatalf("src=%q: expected error, got none", c.src) + } + if !strings.Contains(err.Error(), "ConverterNotFoundError") { + t.Errorf("src=%q: want ConverterNotFoundError, got %v", c.src, err) + } + if !strings.Contains(err.Error(), c.wantSub) { + t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err) + } + }) + } +} + +func TestConverterNewTypeError(t *testing.T) { + for _, src := range []string{ + `Encoding::Converter.new(123, "utf-8")`, + `Encoding::Converter.new("utf-8", nil)`, + } { + err := runErr(t, src) + if err == nil || !strings.Contains(err.Error(), "TypeError") { + t.Errorf("src=%q: want TypeError, got %v", src, err) + } + } +} + +// TestConverterNewValidPair confirms a genuinely deliverable pair still +// constructs (the deliverability check must not over-reject). +func TestConverterNewValidPair(t *testing.T) { + for _, src := range []string{ + `p Encoding::Converter.new("utf-8", "utf-16le").class.name`, + `p Encoding::Converter.new("euc-jp", "utf-8").class.name`, + `p Encoding::Converter.new(Encoding::UTF_8, Encoding::SHIFT_JIS).class.name`, + } { + if got := eval(t, src); got != "\"Encoding::Converter\"\n" { + t.Errorf("src=%q: got %q", src, got) + } + } +} + +// TestConcatAliasedReceiver covers the snapshot fix for a receiver that also +// appears among its own concat arguments: MRI contributes the receiver's INITIAL +// value at each position. +func TestConcatAliasedReceiver(t *testing.T) { + cases := []struct{ src, want string }{ + {`a = "hello"; a.concat(a, a); p a`, "\"hellohellohello\"\n"}, + {`a = "ab"; a.concat(a); p a`, "\"abab\"\n"}, // single arg, one append + {`a = "x"; a.concat("y", a, "z"); p a`, "\"xyxz\"\n"}, // mixed, self in the middle + {`p "ab".concat("c", "d")`, "\"abcd\"\n"}, // no aliasing, multi-arg + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } +} + +// TestConcatMultibyteCodepoint covers appending an Integer codepoint to a string +// in a legacy multibyte encoding: a valid codepoint is unpacked big-endian, an +// in-range but non-character byte (a lone lead byte) raises RangeError, and +// single-byte encodings still accept any 0..255 byte. +func TestConcatMultibyteCodepoint(t *testing.T) { + ok := []struct{ src, want string }{ + {`s = "".force_encoding("euc-jp"); s.concat(0xa4a2); p s.bytes`, "[164, 162]\n"}, + {`s = "".force_encoding("shift_jis"); s.concat(0x82a0); p s.bytes`, "[130, 160]\n"}, + {`p 0xa4a2.chr("euc-jp").bytes`, "[164, 162]\n"}, + {`s = "".force_encoding("iso-8859-1"); s.concat(0x81); p s.bytes`, "[129]\n"}, + } + for _, c := range ok { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } + bad := []struct{ src, wantSub string }{ + {`"".force_encoding("euc-jp").concat(0x81)`, "invalid codepoint 0x81 in EUC-JP"}, + {`"".force_encoding("shift_jis").concat(0x81)`, "invalid codepoint 0x81 in Shift_JIS"}, + {`0x81.chr("euc-jp")`, "invalid codepoint 0x81 in EUC-JP"}, + {`"".force_encoding("euc-jp").concat(-1)`, "invalid codepoint"}, + {`"".force_encoding("us-ascii").concat(256)`, "out of char range"}, + } + for _, c := range bad { + err := runErr(t, c.src) + if err == nil || !strings.Contains(err.Error(), "RangeError") { + t.Errorf("src=%q: want RangeError, got %v", c.src, err) + continue + } + if !strings.Contains(err.Error(), c.wantSub) { + t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err) + } + } +} From f8e809731e8d317d6670e86caa0c117f100ab291 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 31 Aug 2026 15:54:18 +0200 Subject: [PATCH 3/4] vm: cover to_str endpoint and zero-codepoint paths in encoding tests Adds encoding_compat_test cases so every branch of the new converterEncName (#to_str resolving to a known and to an unknown name) and bigEndianBytes (codepoint 0) is exercised, keeping the changed functions at 100%. Co-Authored-By: Claude Opus 4.8 --- internal/vm/encoding_compat_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/vm/encoding_compat_test.go b/internal/vm/encoding_compat_test.go index 590d74ec..51bb635b 100644 --- a/internal/vm/encoding_compat_test.go +++ b/internal/vm/encoding_compat_test.go @@ -64,12 +64,14 @@ func TestConverterNewTypeError(t *testing.T) { } // TestConverterNewValidPair confirms a genuinely deliverable pair still -// constructs (the deliverability check must not over-reject). +// constructs (the deliverability check must not over-reject). The final case +// resolves an endpoint through #to_str. func TestConverterNewValidPair(t *testing.T) { for _, src := range []string{ `p Encoding::Converter.new("utf-8", "utf-16le").class.name`, `p Encoding::Converter.new("euc-jp", "utf-8").class.name`, `p Encoding::Converter.new(Encoding::UTF_8, Encoding::SHIFT_JIS).class.name`, + `o = Object.new; def o.to_str; "utf-16le"; end; p Encoding::Converter.new(o, "utf-8").class.name`, } { if got := eval(t, src); got != "\"Encoding::Converter\"\n" { t.Errorf("src=%q: got %q", src, got) @@ -77,6 +79,17 @@ func TestConverterNewValidPair(t *testing.T) { } } +// TestConverterNewToStrUnknown resolves an endpoint through #to_str to an +// unregistered name: the raw #to_str result is kept in the not-found message. +func TestConverterNewToStrUnknown(t *testing.T) { + src := `o = Object.new; def o.to_str; "bogus-enc"; end; Encoding::Converter.new(o, "utf-8")` + err := runErr(t, src) + if err == nil || !strings.Contains(err.Error(), "ConverterNotFoundError") || + !strings.Contains(err.Error(), "code converter not found (bogus-enc to UTF-8)") { + t.Errorf("src=%q: got %v", src, err) + } +} + // TestConcatAliasedReceiver covers the snapshot fix for a receiver that also // appears among its own concat arguments: MRI contributes the receiver's INITIAL // value at each position. @@ -103,6 +116,7 @@ func TestConcatMultibyteCodepoint(t *testing.T) { {`s = "".force_encoding("euc-jp"); s.concat(0xa4a2); p s.bytes`, "[164, 162]\n"}, {`s = "".force_encoding("shift_jis"); s.concat(0x82a0); p s.bytes`, "[130, 160]\n"}, {`p 0xa4a2.chr("euc-jp").bytes`, "[164, 162]\n"}, + {`p 0.chr("euc-jp").bytes`, "[0]\n"}, // codepoint 0 unpacks to a single NUL byte {`s = "".force_encoding("iso-8859-1"); s.concat(0x81); p s.bytes`, "[129]\n"}, } for _, c := range ok { From a166942fa542ae3972d041e5367cf970ac2f4dcf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Mon, 31 Aug 2026 15:57:35 +0200 Subject: [PATCH 4/4] vm: String#ascii_only? is false for a non-ASCII-compatible encoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #ascii_only? reports MRI's ENC_CODERANGE_7BIT, not a bare byte scan: a string tagged with a non-ASCII-compatible encoding (UTF-16/32, …) is never 7-bit even when empty or all-ASCII in bytes, so it now returns false there instead of scanning only the bytes. ASCII-compatible encodings keep the byte-scan result. Fixes the two ascii_only_spec examples for empty and non-empty strings in a non-ASCII-compatible encoding. Co-Authored-By: Claude Opus 4.8 --- internal/vm/encoding.go | 8 ++++++-- internal/vm/encoding_compat_test.go | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/internal/vm/encoding.go b/internal/vm/encoding.go index 2a47dcf9..cf3c6033 100644 --- a/internal/vm/encoding.go +++ b/internal/vm/encoding.go @@ -384,8 +384,12 @@ func (vm *VM) registerStringEncoding() { d.Enc = "ASCII-8BIT" return d }) - vm.cString.define("ascii_only?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - return object.Bool(asciiOnly(self.(*object.String).Bytes())) + vm.cString.define("ascii_only?", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + // #ascii_only? is ENC_CODERANGE_7BIT: content in a non-ASCII-compatible + // encoding (UTF-16/32, …) is never 7-bit, even when empty or all-ASCII in + // bytes, so it reports false regardless of the bytes. + s := self.(*object.String) + return object.Bool(vm.internEncoding(s.EncName()).asciiCompat && asciiOnly(s.Bytes())) }) vm.cString.define("valid_encoding?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { s := self.(*object.String) diff --git a/internal/vm/encoding_compat_test.go b/internal/vm/encoding_compat_test.go index 51bb635b..add71579 100644 --- a/internal/vm/encoding_compat_test.go +++ b/internal/vm/encoding_compat_test.go @@ -107,6 +107,24 @@ func TestConcatAliasedReceiver(t *testing.T) { } } +// TestAsciiOnlyEncoding covers that #ascii_only? is the 7-bit coderange, not a +// bare byte scan: content in a non-ASCII-compatible encoding is never 7-bit, +// even when empty or all-ASCII in bytes. +func TestAsciiOnlyEncoding(t *testing.T) { + cases := []struct{ src, want string }{ + {`p "abc".ascii_only?`, "true\n"}, + {`p "".ascii_only?`, "true\n"}, // empty UTF-8 is 7-bit + {`p "".force_encoding("UTF-16LE").ascii_only?`, "false\n"}, + {`p "abc".force_encoding("UTF-16BE").ascii_only?`, "false\n"}, + {`p "\xe3\x81\x82".ascii_only?`, "false\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q: got %q want %q", c.src, got, c.want) + } + } +} + // TestConcatMultibyteCodepoint covers appending an Integer codepoint to a string // in a legacy multibyte encoding: a valid codepoint is unpacked big-endian, an // in-range but non-character byte (a lone lead byte) raises RangeError, and