Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion internal/vm/builtins.go
Original file line number Diff line number Diff line change
Expand Up @@ -2057,6 +2057,17 @@ func (vm *VM) bootstrap() {
strConcatFn := func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value {
s := self.(*object.String)
vm.checkFrozen(s)
// Freeze any argument that aliases the receiver before the loop mutates it,
// so `a.concat(a, a)` contributes a's INITIAL value at every position (MRI:
// "hellohellohello", not "hellohellohellohello").
if len(args) > 1 {
args = append([]object.Value(nil), args...)
for i, a := range args {
if str, ok := a.(*object.String); ok && str == s {
args[i] = str.Dup()
}
}
}
for _, a := range args {
if u, ok := a.(object.KeyUnwrapper); ok { // a String subclass instance
if w, wrapped := u.HashUnwrap(); wrapped {
Expand Down Expand Up @@ -5224,14 +5235,48 @@ func chrEncode(n int64, enc string) []byte {
raise("RangeError", "%d out of char range", n)
}
return []byte{byte(n)}
default: // ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte
default:
// A legacy multibyte encoding (EUC-JP, Shift_JIS, …) interprets the Integer
// as a big-endian byte sequence that must form exactly one valid character
// (MRI's rb_enc_mbcput + precise-mbclen check); a lone lead byte such as
// 0x81 in EUC-JP is in byte range yet not a character, so it raises
// "invalid codepoint" rather than the plain "out of char range".
if codec, isMB := xtextEncodings[enc]; isMB {
if n >= 0 {
be := bigEndianBytes(uint64(n))
if out, err := codec.NewDecoder().Bytes(be); err == nil &&
utf8.RuneCount(out) == 1 && !strings.ContainsRune(string(out), utf8.RuneError) {
return be
}
}
raise("RangeError", "invalid codepoint 0x%x in %s", n, enc)
}
// ASCII-8BIT/BINARY and single-byte encodings hold one 0..255 byte.
if n < 0 || n > 255 {
raise("RangeError", "%d out of char range", n)
}
return []byte{byte(n)}
}
}

// bigEndianBytes returns n as its minimal big-endian byte sequence (a single
// zero byte for 0). It is how an Integer codepoint is unpacked into the encoded
// bytes for a legacy multibyte encoding.
func bigEndianBytes(n uint64) []byte {
if n == 0 {
return []byte{0}
}
var b []byte
for n > 0 {
b = append(b, byte(n))
n >>= 8
}
for i, j := 0, len(b)-1; i < j; i, j = i+1, j-1 {
b[i], b[j] = b[j], b[i]
}
return b
}

// codepointAppend encodes the codepoint n for `String#<<`/#concat with an Integer
// argument, returning its bytes and the resulting string encoding. It mirrors
// chrEncode but, like MRI, promotes a US-ASCII receiver to ASCII-8BIT when the
Expand Down
58 changes: 50 additions & 8 deletions internal/vm/converter_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,16 @@ func (vm *VM) converterNew(args []object.Value) object.Value {
if len(args) < 2 {
raise("ArgumentError", "wrong number of arguments (given %d, expected 2..3)", len(args))
}
c := &converterObj{
src: vm.encodingArg(args[0]).name,
dst: vm.encodingArg(args[1]).name,
status: "source_buffer_empty",
}
if c.src == c.dst {
raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", c.src, c.dst)
}
// MRI resolves both endpoints through rb_econv_open: an unknown name, an
// identical src/dst, or a pair with no available codec all raise
// ConverterNotFoundError (never ArgumentError). An unresolvable name is kept
// verbatim in the message; a resolvable one is canonicalised.
src, srcOK := vm.converterEncName(args[0])
dst, dstOK := vm.converterEncName(args[1])
if !srcOK || !dstOK || src == dst || !vm.converterDeliverable(src, dst) {
raise("Encoding::ConverterNotFoundError", "code converter not found (%s to %s)", src, dst)
}
c := &converterObj{src: src, dst: dst, status: "source_buffer_empty"}
c.repl, c.replEnc = defaultReplacementBytes(c.dst)

if len(args) >= 3 {
Expand All @@ -53,6 +55,46 @@ func (vm *VM) converterNew(args []object.Value) object.Value {
return c
}

// converterEncName resolves one Converter.new endpoint to a canonical encoding
// name. An Encoding, or a String / #to_str naming a registered encoding, yields
// (canonicalName, true); an unregistered name yields the raw string with
// ok=false, so the caller raises ConverterNotFoundError keeping the name exactly
// as given (as MRI does). A non-String, non-Encoding raises TypeError.
func (vm *VM) converterEncName(v object.Value) (string, bool) {
switch e := v.(type) {
case *encodingObj:
return e.name, true
case *object.String:
if enc, ok := vm.findEncoding(e.Str()); ok {
return enc.name, true
}
return e.Str(), false
}
if vm.respondsToDynamic(v, "to_str") {
if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok {
if enc, ok := vm.findEncoding(s.Str()); ok {
return enc.name, true
}
return s.Str(), false
}
}
raise("TypeError", "no implicit conversion of %s into String", classNameOf(v))
return "", false
}

// converterDeliverable reports whether rbgo can build every hop of a src→dst
// conversion (both names already canonical and distinct). It mirrors the hop
// check buildConvpath performs, without allocating the path Array.
func (vm *VM) converterDeliverable(src, dst string) bool {
path := pathList(src, dst)
for i := 0; i+1 < len(path); i++ {
if !vm.hopConvertible(path[i], path[i+1]) {
return false
}
}
return true
}

// applyConverterOpts reads Converter.new's third argument: an options Hash (its
// keys drive the flags), or an Integer whose INVALID_REPLACE / UNDEF_REPLACE /
// CRLF_NEWLINE_DECORATOR bits set the same flags.
Expand Down
6 changes: 3 additions & 3 deletions internal/vm/converter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,9 @@ func TestConverterConvert(t *testing.T) {
{`c = Encoding::Converter.new("utf-8","iso-8859-1"); c.finish; c.convert("a")`, "convert after finish"},
// A trailing incomplete sequence buffers, then #finish raises.
{`c = Encoding::Converter.new("euc-jp","iso-8859-1"); c.convert("\xa4".b); c.finish`, "incomplete"},
// Destination without a codec: even ASCII is undefined.
{`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "UndefinedConversionError"},
// Destination without a codec: MRI rejects the pair at construction time
// (rb_econv_open) rather than at convert time.
{`Encoding::Converter.new("utf-8","UTF-7").convert("a")`, "ConverterNotFoundError"},
{`Encoding::Converter.new("utf-8","shift_jis").convert("😀")`, "UndefinedConversionError"},
}
for _, c := range errCases {
Expand Down Expand Up @@ -210,7 +211,6 @@ func TestConverterPrimitive(t *testing.T) {
// x/text source fallback: a bare invalid high byte and a truncated lead.
{`c = Encoding::Converter.new("shift_jis","utf-8"); s = "\xfd\xfd\xfd\xfd".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"},
{`c = Encoding::Converter.new("shift_jis","utf-8"); p c.primitive_convert("\x82".b, String.new, nil, nil, partial_input: true)`, ":source_buffer_empty\n"},
{`c = Encoding::Converter.new("UTF-7","utf-8"); s = "\x80".b; p c.primitive_convert(s, String.new)`, ":invalid_byte_sequence\n"},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want {
Expand Down
8 changes: 6 additions & 2 deletions internal/vm/encoding.go
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,12 @@ func (vm *VM) registerStringEncoding() {
d.Enc = "ASCII-8BIT"
return d
})
vm.cString.define("ascii_only?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
return object.Bool(asciiOnly(self.(*object.String).Bytes()))
vm.cString.define("ascii_only?", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
// #ascii_only? is ENC_CODERANGE_7BIT: content in a non-ASCII-compatible
// encoding (UTF-16/32, …) is never 7-bit, even when empty or all-ASCII in
// bytes, so it reports false regardless of the bytes.
s := self.(*object.String)
return object.Bool(vm.internEncoding(s.EncName()).asciiCompat && asciiOnly(s.Bytes()))
})
vm.cString.define("valid_encoding?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
s := self.(*object.String)
Expand Down
162 changes: 162 additions & 0 deletions internal/vm/encoding_compat_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
package vm_test

import (
"strings"
"testing"
)

// TestConverterNewNotFound covers Encoding::Converter.new's endpoint resolution.
// MRI's rb_econv_open raises Encoding::ConverterNotFoundError — never
// ArgumentError — for an unknown name, an identical src/dst, or a known but
// undeliverable pair (any pair through the dummy UTF-7). A non-String,
// non-Encoding argument raises TypeError. Every expectation is verified against
// MRI 4.0.x.
func TestConverterNewNotFound(t *testing.T) {
cases := []struct {
name, src, wantSub string
}{
// Unknown source keeps its name verbatim; the known destination is
// canonicalised ("utf-8" -> "UTF-8").
{"unknown_src", `Encoding::Converter.new("foo-bar", "utf-8")`,
"code converter not found (foo-bar to UTF-8)"},
{"unknown_dst", `Encoding::Converter.new("utf-8", "foo-bar")`,
"code converter not found (UTF-8 to foo-bar)"},
{"both_unknown", `Encoding::Converter.new("foo", "bar")`,
"code converter not found (foo to bar)"},
// Known but undeliverable: the dummy UTF-7 has no codec in either direction.
{"undeliverable_to_utf7", `Encoding::Converter.new("utf-8", "utf-7")`,
"code converter not found (UTF-8 to UTF-7)"},
{"undeliverable_from_utf7", `Encoding::Converter.new("utf-7", "utf-8")`,
"code converter not found (UTF-7 to UTF-8)"},
// Identical endpoints are their own not-found case.
{"identical", `Encoding::Converter.new("utf-8", "utf-8")`,
"code converter not found (UTF-8 to UTF-8)"},
// An Encoding object resolves like its canonical name.
{"encoding_object", `Encoding::Converter.new(Encoding::UTF_8, Encoding::UTF_7)`,
"code converter not found (UTF-8 to UTF-7)"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
err := runErr(t, c.src)
if err == nil {
t.Fatalf("src=%q: expected error, got none", c.src)
}
if !strings.Contains(err.Error(), "ConverterNotFoundError") {
t.Errorf("src=%q: want ConverterNotFoundError, got %v", c.src, err)
}
if !strings.Contains(err.Error(), c.wantSub) {
t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err)
}
})
}
}

func TestConverterNewTypeError(t *testing.T) {
for _, src := range []string{
`Encoding::Converter.new(123, "utf-8")`,
`Encoding::Converter.new("utf-8", nil)`,
} {
err := runErr(t, src)
if err == nil || !strings.Contains(err.Error(), "TypeError") {
t.Errorf("src=%q: want TypeError, got %v", src, err)
}
}
}

// TestConverterNewValidPair confirms a genuinely deliverable pair still
// constructs (the deliverability check must not over-reject). The final case
// resolves an endpoint through #to_str.
func TestConverterNewValidPair(t *testing.T) {
for _, src := range []string{
`p Encoding::Converter.new("utf-8", "utf-16le").class.name`,
`p Encoding::Converter.new("euc-jp", "utf-8").class.name`,
`p Encoding::Converter.new(Encoding::UTF_8, Encoding::SHIFT_JIS).class.name`,
`o = Object.new; def o.to_str; "utf-16le"; end; p Encoding::Converter.new(o, "utf-8").class.name`,
} {
if got := eval(t, src); got != "\"Encoding::Converter\"\n" {
t.Errorf("src=%q: got %q", src, got)
}
}
}

// TestConverterNewToStrUnknown resolves an endpoint through #to_str to an
// unregistered name: the raw #to_str result is kept in the not-found message.
func TestConverterNewToStrUnknown(t *testing.T) {
src := `o = Object.new; def o.to_str; "bogus-enc"; end; Encoding::Converter.new(o, "utf-8")`
err := runErr(t, src)
if err == nil || !strings.Contains(err.Error(), "ConverterNotFoundError") ||
!strings.Contains(err.Error(), "code converter not found (bogus-enc to UTF-8)") {
t.Errorf("src=%q: got %v", src, err)
}
}

// TestConcatAliasedReceiver covers the snapshot fix for a receiver that also
// appears among its own concat arguments: MRI contributes the receiver's INITIAL
// value at each position.
func TestConcatAliasedReceiver(t *testing.T) {
cases := []struct{ src, want string }{
{`a = "hello"; a.concat(a, a); p a`, "\"hellohellohello\"\n"},
{`a = "ab"; a.concat(a); p a`, "\"abab\"\n"}, // single arg, one append
{`a = "x"; a.concat("y", a, "z"); p a`, "\"xyxz\"\n"}, // mixed, self in the middle
{`p "ab".concat("c", "d")`, "\"abcd\"\n"}, // no aliasing, multi-arg
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want {
t.Errorf("src=%q: got %q want %q", c.src, got, c.want)
}
}
}

// TestAsciiOnlyEncoding covers that #ascii_only? is the 7-bit coderange, not a
// bare byte scan: content in a non-ASCII-compatible encoding is never 7-bit,
// even when empty or all-ASCII in bytes.
func TestAsciiOnlyEncoding(t *testing.T) {
cases := []struct{ src, want string }{
{`p "abc".ascii_only?`, "true\n"},
{`p "".ascii_only?`, "true\n"}, // empty UTF-8 is 7-bit
{`p "".force_encoding("UTF-16LE").ascii_only?`, "false\n"},
{`p "abc".force_encoding("UTF-16BE").ascii_only?`, "false\n"},
{`p "\xe3\x81\x82".ascii_only?`, "false\n"},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want {
t.Errorf("src=%q: got %q want %q", c.src, got, c.want)
}
}
}

// TestConcatMultibyteCodepoint covers appending an Integer codepoint to a string
// in a legacy multibyte encoding: a valid codepoint is unpacked big-endian, an
// in-range but non-character byte (a lone lead byte) raises RangeError, and
// single-byte encodings still accept any 0..255 byte.
func TestConcatMultibyteCodepoint(t *testing.T) {
ok := []struct{ src, want string }{
{`s = "".force_encoding("euc-jp"); s.concat(0xa4a2); p s.bytes`, "[164, 162]\n"},
{`s = "".force_encoding("shift_jis"); s.concat(0x82a0); p s.bytes`, "[130, 160]\n"},
{`p 0xa4a2.chr("euc-jp").bytes`, "[164, 162]\n"},
{`p 0.chr("euc-jp").bytes`, "[0]\n"}, // codepoint 0 unpacks to a single NUL byte
{`s = "".force_encoding("iso-8859-1"); s.concat(0x81); p s.bytes`, "[129]\n"},
}
for _, c := range ok {
if got := eval(t, c.src); got != c.want {
t.Errorf("src=%q: got %q want %q", c.src, got, c.want)
}
}
bad := []struct{ src, wantSub string }{
{`"".force_encoding("euc-jp").concat(0x81)`, "invalid codepoint 0x81 in EUC-JP"},
{`"".force_encoding("shift_jis").concat(0x81)`, "invalid codepoint 0x81 in Shift_JIS"},
{`0x81.chr("euc-jp")`, "invalid codepoint 0x81 in EUC-JP"},
{`"".force_encoding("euc-jp").concat(-1)`, "invalid codepoint"},
{`"".force_encoding("us-ascii").concat(256)`, "out of char range"},
}
for _, c := range bad {
err := runErr(t, c.src)
if err == nil || !strings.Contains(err.Error(), "RangeError") {
t.Errorf("src=%q: want RangeError, got %v", c.src, err)
continue
}
if !strings.Contains(err.Error(), c.wantSub) {
t.Errorf("src=%q: want message containing %q, got %v", c.src, c.wantSub, err)
}
}
}