From 70ecb26ed7c30c87139bc7de5f8976fb89e4c890 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 6 Sep 2026 10:15:18 +0200 Subject: [PATCH 1/3] Encoding: dynamic find names, default_external= nil, macCyrillic, putback errinfo Advance MRI 4.0.6 conformance in the encoding subsystem: - Encoding.find("external"/"filesystem"/"locale") now resolves to the current Encoding.default_external, and "internal" to Encoding.default_internal (nil when unset), tracking the setters instead of the pinned UTF-8 aliases. - Encoding.default_external= rejects nil with ArgumentError ("default external can not be nil") rather than the generic coercion TypeError, and stores the default on the VM so find() reflects it. - Add the macCyrillic (Macintosh Cyrillic) codec via golang.org/x/text so String#encode and Encoding::Converter can transcode UTF-8<->macCyrillic. - Encoding::Converter#putback no longer overwrites the read-again bytes that #primitive_errinfo reports, matching MRI where putback and primitive_errinfo are independent. Specs: default_external 5->8, find 11->12 (error cleared), converter/convert 5->7 (2 errors cleared), converter/putback 6->7. Co-Authored-By: Claude Opus 4.8 --- internal/vm/converter_impl.go | 3 +- internal/vm/encoding.go | 30 +++++++-- internal/vm/encoding_residuals_wave13_test.go | 63 +++++++++++++++++++ internal/vm/string_encode_xtext.go | 1 + internal/vm/vm.go | 1 + 5 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 internal/vm/encoding_residuals_wave13_test.go diff --git a/internal/vm/converter_impl.go b/internal/vm/converter_impl.go index f5787d87..6da8a030 100644 --- a/internal/vm/converter_impl.go +++ b/internal/vm/converter_impl.go @@ -570,7 +570,8 @@ func (vm *VM) converterPutback(c *converterObj, args []object.Value) object.Valu b := c.readAgain[len(c.readAgain)-n:] out := object.NewStringBytesEnc(append([]byte(nil), b...), c.src) c.readAgain = c.readAgain[:len(c.readAgain)-n] - c.errReadAgain = c.readAgain + // #putback does not disturb #primitive_errinfo: MRI keeps reporting the + // original read-again bytes after a putback, so c.errReadAgain is left as set. return out } diff --git a/internal/vm/encoding.go b/internal/vm/encoding.go index cf3c6033..69c23b3f 100644 --- a/internal/vm/encoding.go +++ b/internal/vm/encoding.go @@ -125,15 +125,21 @@ func (vm *VM) registerEncoding() { // values Puppet's log_runtime_environment and IO setup read. The setters are // accepted (and remembered) so code that brackets work in an encoding override // runs; rbgo's string layer remains UTF-8 regardless. - defExternal := vm.encodings["UTF-8"] + vm.defExternalEnc = vm.encodings["UTF-8"] sdef := func(name string, fn NativeFn) { vm.cEncoding.smethods[name] = &Method{name: name, owner: vm.cEncoding, native: fn} } - sdef("default_external", func(_ *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value { - return defExternal + sdef("default_external", func(vm *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value { + return vm.defExternalEnc }) sdef("default_external=", func(vm *VM, _ object.Value, args []object.Value, _ *Proc) object.Value { - defExternal = vm.encodingArg(args[0]) + // MRI rejects nil with ArgumentError (not the TypeError encodingArg would + // raise), and setting the default also redirects the "external", + // "filesystem" and "locale" names Encoding.find resolves. + if _, isNil := args[0].(object.Nil); isNil { + raise("ArgumentError", "default external can not be nil") + } + vm.defExternalEnc = vm.encodingArg(args[0]) return args[0] }) sdef("default_internal", func(vm *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value { @@ -157,8 +163,22 @@ func (vm *VM) registerEncoding() { return object.NewString("UTF-8") }) // Encoding.find(name) resolves a name (or a pass-through Encoding) to the - // registry object, raising ArgumentError for an unknown name as MRI does. + // registry object, raising ArgumentError for an unknown name as MRI does. The + // dynamic names track the current process defaults: "external"/"filesystem"/ + // "locale" are Encoding.default_external, and "internal" is + // Encoding.default_internal (nil when unset). sdef("find", func(vm *VM, _ object.Value, args []object.Value, _ *Proc) object.Value { + if s, ok := args[0].(*object.String); ok { + switch lower(s.Str()) { + case "external", "filesystem", "locale": + return vm.defExternalEnc + case "internal": + if vm.defInternalEnc == nil { + return object.NilV + } + return vm.defInternalEnc + } + } return vm.encodingArg(args[0]) }) // Encoding.list returns every registered (non-alias) encoding, in registration diff --git a/internal/vm/encoding_residuals_wave13_test.go b/internal/vm/encoding_residuals_wave13_test.go new file mode 100644 index 00000000..7f5f481c --- /dev/null +++ b/internal/vm/encoding_residuals_wave13_test.go @@ -0,0 +1,63 @@ +package vm_test + +import ( + "strings" + "testing" +) + +// TestEncodingResidualsWave13 covers the encoding-subsystem conformance fixes of +// wave 13: the dynamic Encoding.find names ("external"/"filesystem"/"locale" -> +// default_external, "internal" -> default_internal), Encoding.default_external= +// rejecting nil, and the macCyrillic (Macintosh Cyrillic) transcoding codec. +// Every expectation is asserted against MRI Ruby 4.0.6. +func TestEncodingResidualsWave13(t *testing.T) { + cases := []struct{ name, src, want string }{ + // Encoding.find resolves the dynamic external/filesystem/locale names to the + // current default_external, tracking the setter. + {"find_external_default", `p Encoding.find("external").name`, "\"UTF-8\"\n"}, + {"find_locale_default", `p Encoding.find("locale").name`, "\"UTF-8\"\n"}, + {"find_filesystem_default", `p Encoding.find("filesystem").name`, "\"UTF-8\"\n"}, + {"find_external_tracks_setter", + `Encoding.default_external = Encoding::SHIFT_JIS +p [Encoding.find("external").name, Encoding.find("filesystem").name]`, + "[\"Shift_JIS\", \"Shift_JIS\"]\n"}, + {"find_external_case_insensitive", `p Encoding.find("EXTERNAL").name`, "\"UTF-8\"\n"}, + // Encoding.find("internal") is default_internal: nil when unset, the encoding + // once set. + {"find_internal_nil", `p Encoding.find("internal")`, "nil\n"}, + {"find_internal_set", + `Encoding.default_internal = Encoding::US_ASCII +p Encoding.find("internal").name`, + "\"US-ASCII\"\n"}, + // macCyrillic (Macintosh Cyrillic) transcodes via golang.org/x/text: a Cyrillic + // letter encodes to its single mac byte, an undefined character raises. + {"maccyrillic_encode", `p "А".encode("macCyrillic").bytes`, "[128]\n"}, + {"maccyrillic_converter", + `ec = Encoding::Converter.new("UTF-8", Encoding.find("macCyrillic")) +p ec.convert("А").bytes`, + "[128]\n"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := eval(t, tc.src); got != tc.want { + t.Errorf("src=%q\n got=%q\nwant=%q", tc.src, got, tc.want) + } + }) + } + + // Error paths: default_external = nil raises ArgumentError (not the TypeError the + // generic coercion would give), and an undefined macCyrillic character raises + // Encoding::UndefinedConversionError. + errCases := []struct{ name, src, want string }{ + {"default_external_nil", `Encoding.default_external = nil`, "default external can not be nil"}, + {"maccyrillic_undef", `"\u{6543}".encode("macCyrillic")`, "from UTF-8 to macCyrillic"}, + } + for _, tc := range errCases { + t.Run(tc.name, func(t *testing.T) { + err := runErr(t, tc.src) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("src=%q: got err=%v, want containing %q", tc.src, err, tc.want) + } + }) + } +} diff --git a/internal/vm/string_encode_xtext.go b/internal/vm/string_encode_xtext.go index 36ebe27b..d29f2674 100644 --- a/internal/vm/string_encode_xtext.go +++ b/internal/vm/string_encode_xtext.go @@ -50,6 +50,7 @@ var xtextEncodings = map[string]encoding.Encoding{ "GB18030": simplifiedchinese.GB18030, "Big5": traditionalchinese.Big5, "macRoman": charmap.Macintosh, + "macCyrillic": charmap.MacintoshCyrillic, } // xtextDecode decodes src (in encoding `from`) to a UTF-8 string using x/text. diff --git a/internal/vm/vm.go b/internal/vm/vm.go index e0d06c9c..e8195a00 100644 --- a/internal/vm/vm.go +++ b/internal/vm/vm.go @@ -404,6 +404,7 @@ type VM struct { encodings map[string]*encodingObj // canonical name -> interned Encoding encLookup map[string]*encodingObj // lowercased name/alias -> interned Encoding defInternalEnc *encodingObj // Encoding.default_internal (nil = MRI's default) + defExternalEnc *encodingObj // Encoding.default_external (UTF-8 unless reset) cLazy *RClass lastMatch object.Value // $~: last regexp MatchData (or nil) regexpTimeout object.Value // Regexp.timeout default (Float seconds or nil) From 7419f1f6fd539ef3e6701b6a814d1c2bee46090b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 6 Sep 2026 10:25:08 +0200 Subject: [PATCH 2/3] Encoding::Converter: model output-buffer carryover and add UTF8-MAC destination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - #primitive_convert now holds converted output that does not fit the destination (MRI's internal output-buffer carryover) in converterObj.pendingOut and flushes it on the next call. stepLoop returns the overflow bytes and, when a character does not fit the byte cap, writes the prefix that fits, holds the remainder and consumes the character — so a full destination returns :destination_buffer_full with the source consumed, and successive capped calls accumulate correctly (the "aabbb" offset behaviour). - Add UTF8-MAC (HFS+ NFD) as a Converter destination: characters are encoded in canonical decomposition via golang.org/x/text/unicode/norm, so UTF-8 -> UTF8-MAC transcoding (and its invalid-byte handling) works. Specs: converter/primitive_convert 27p/3f/2e -> 32p/0f/0e (offset carryover, two destination_buffer_full source-clearing cases, two UTF8-MAC cases). Encoding cluster now 596 pass / 3 fail / 0 error. Co-Authored-By: Claude Opus 4.8 --- internal/vm/converter.go | 10 +-- internal/vm/converter_codec.go | 6 ++ internal/vm/converter_impl.go | 71 +++++++++++++++---- internal/vm/encoding_residuals_wave13_test.go | 32 +++++++++ 4 files changed, 101 insertions(+), 18 deletions(-) diff --git a/internal/vm/converter.go b/internal/vm/converter.go index 2214d1bf..fe18cc97 100644 --- a/internal/vm/converter.go +++ b/internal/vm/converter.go @@ -16,10 +16,11 @@ import ( // codecs), driving them through a byte-at-a-time conversion loop so the // #primitive_convert state machine (status symbols, source-buffer consumption, // #primitive_errinfo, #last_error, #putback) matches MRI for the common single -// conversion step. The residual — MRI's internal output-buffer carryover across -// #primitive_convert calls (the "� spills into the next buffer" behaviour) -// and the multi-hop stateful decorators (ISO-2022-JP escape-state #finish) — is -// documented at the methods that would need it. +// conversion step. Output that does not fit a #primitive_convert destination is +// held in pendingOut and flushed on the next call (MRI's internal output-buffer +// carryover). The remaining residual — the multi-hop stateful decorators +// (ISO-2022-JP escape-state #finish) — is documented at the methods that would +// need it. type converterObj struct { src, dst string // canonical source / destination encoding names @@ -41,6 +42,7 @@ type converterObj struct { lastErrExc object.Value pendingIncomplete []byte // incomplete tail buffered by #convert for #finish + pendingOut []byte // converted output that did not fit the destination, flushed on the next #primitive_convert finished bool // #finish has been called (stream closed) } diff --git a/internal/vm/converter_codec.go b/internal/vm/converter_codec.go index 0e0bf785..ee87cb0c 100644 --- a/internal/vm/converter_codec.go +++ b/internal/vm/converter_codec.go @@ -7,6 +7,8 @@ package vm import ( binpkg "encoding/binary" "unicode/utf8" + + "golang.org/x/text/unicode/norm" ) // decodeCharFrom decodes the next character of in, interpreted in encoding @@ -223,6 +225,10 @@ func (vm *VM) encodeCharTo(r rune, to string) ([]byte, bool) { return encodeUTF32(string(r), false), true case "UTF-32BE": return encodeUTF32(string(r), true), true + case "UTF8-MAC": + // UTF8-MAC (a.k.a. UTF-8-MAC / HFS+) is UTF-8 in canonical decomposition + // (NFD), so each character is encoded as its NFD bytes. + return norm.NFD.Bytes([]byte(string(r))), true } enc, ok := xtextEncodings[to] if !ok { diff --git a/internal/vm/converter_impl.go b/internal/vm/converter_impl.go index 6da8a030..c86e3ca4 100644 --- a/internal/vm/converter_impl.go +++ b/internal/vm/converter_impl.go @@ -256,7 +256,8 @@ func (vm *VM) hopConvertible(from, to string) bool { func encodingSupported(name string) bool { switch name { case "UTF-8", "US-ASCII", "ASCII-8BIT", "ISO-8859-1", - "UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE": + "UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE", + "UTF8-MAC": return true } _, ok := xtextEncodings[name] @@ -324,7 +325,7 @@ func (vm *VM) converterConvert(c *converterObj, v object.Value) object.Value { } in := append(append([]byte(nil), c.readAgain...), vm.strToStr(v)...) c.readAgain = nil - out, _, status, errBytes, readAgain := vm.stepLoop(c, in, -1) + out, _, status, errBytes, readAgain, _ := vm.stepLoop(c, in, -1) switch status { case "invalid_byte_sequence": vm.setConvError(c, "invalid_byte_sequence", errBytes, readAgain, false) @@ -406,7 +407,40 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu } } - out, consumed, status, errBytes, readAgain := vm.stepLoop(c, in, size) + // Output held from a previous destination_buffer_full is flushed first; only the + // remaining destination room is offered to the conversion of new source. + carried := c.pendingOut + c.pendingOut = nil + room := size + if size >= 0 { + room = size - len(carried) + if room < 0 { + room = 0 + } + } + + out, consumed, status, errBytes, readAgain, overflow := vm.stepLoop(c, in, room) + + // Combine the carried output with this pass's output, then apply the destination + // cap: bytes past it (plus any stepLoop overflow) are held for the next call. + produced := append(append([]byte(nil), carried...), out...) + var hold []byte + if size >= 0 && len(produced) > size { + hold = append(hold, produced[size:]...) + produced = produced[:size] + } + hold = append(hold, overflow...) + if len(hold) > 0 { + c.pendingOut = hold + // Unwritten output means the destination is full — unless the pass stopped on + // a source error, which takes priority and keeps its bytes unconsumed. + switch status { + case "invalid_byte_sequence", "undefined_conversion", "incomplete_input": + default: + status = "destination_buffer_full" + } + } + if partial && status == "incomplete_input" { status = "source_buffer_empty" } else if !partial && status == "source_buffer_empty" { @@ -419,7 +453,7 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu if offset < len(base) { base = base[:offset] } - dst.SetBytes(append(append([]byte(nil), base...), out...)) + dst.SetBytes(append(append([]byte(nil), base...), produced...)) dst.Enc = c.dst // Consume the converted bytes from the source String (when one was given). @@ -445,11 +479,15 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu // stepLoop is the shared conversion engine for #convert and #primitive_convert. // It decodes src character by character in the source encoding, encodes each into -// the destination encoding, and stops at the first error or when appending the -// next character would exceed size (a byte cap; negative means unlimited). It +// the destination encoding, and stops at the first error or when a character's +// encoded bytes no longer fit within size (a byte cap; negative means unlimited). +// When a character overflows the cap the bytes that fit go to out, the remainder +// to overflow, that character is consumed, and the loop stops with +// "destination_buffer_full" — the caller carries overflow to the next call. It // returns the produced bytes, the number of source bytes consumed, a status -// string, and the erroneous / read-again bytes for the error statuses. -func (vm *VM) stepLoop(c *converterObj, in []byte, size int) (out []byte, consumed int, status string, errBytes, readAgain []byte) { +// string, the erroneous / read-again bytes for the error statuses, and the +// overflow bytes. +func (vm *VM) stepLoop(c *converterObj, in []byte, size int) (out []byte, consumed int, status string, errBytes, readAgain, overflow []byte) { pos := 0 for pos < len(in) { r, n, rl, st := vm.decodeCharFrom(in[pos:], c.src) @@ -457,32 +495,37 @@ func (vm *VM) stepLoop(c *converterObj, in []byte, size int) (out []byte, consum case stepIncomplete: if c.invalidReplace { out = appendCapped(out, c.repl, size) - return out, len(in), "finished", nil, nil + return out, len(in), "finished", nil, nil, nil } - return out, len(in), "incomplete_input", in[pos:], nil + return out, len(in), "incomplete_input", in[pos:], nil, nil case stepInvalid: if c.invalidReplace { out = appendCapped(out, c.repl, size) pos += n + rl continue } - return out, pos + n + rl, "invalid_byte_sequence", in[pos : pos+n], in[pos+n : pos+n+rl] + return out, pos + n + rl, "invalid_byte_sequence", in[pos : pos+n], in[pos+n : pos+n+rl], nil } eb, okEnc := vm.encodeCharTo(r, c.dst) if !okEnc { if c.undefReplace { eb = c.repl } else { - return out, pos + n, "undefined_conversion", []byte(string(r)), nil + return out, pos + n, "undefined_conversion", []byte(string(r)), nil, nil } } if size >= 0 && len(out)+len(eb) > size { - return out, pos, "destination_buffer_full", nil, nil + // The character does not fit: write the prefix that fits, hold the rest, + // consume the character, and stop with the destination full. + room := size - len(out) + out = append(out, eb[:room]...) + overflow = append(overflow, eb[room:]...) + return out, pos + n, "destination_buffer_full", nil, nil, overflow } out = append(out, eb...) pos += n } - return out, len(in), "source_buffer_empty", nil, nil + return out, len(in), "source_buffer_empty", nil, nil, nil } // appendCapped appends add to out unless doing so would exceed a non-negative diff --git a/internal/vm/encoding_residuals_wave13_test.go b/internal/vm/encoding_residuals_wave13_test.go index 7f5f481c..56e6a042 100644 --- a/internal/vm/encoding_residuals_wave13_test.go +++ b/internal/vm/encoding_residuals_wave13_test.go @@ -36,6 +36,38 @@ p Encoding.find("internal").name`, `ec = Encoding::Converter.new("UTF-8", Encoding.find("macCyrillic")) p ec.convert("А").bytes`, "[128]\n"}, + // Encoding::Converter#primitive_convert carries output that did not fit the + // destination to the next call: successive calls with a growing byte cap flush + // the held byte before converting the next character (MRI's output-buffer + // carryover), so three one-byte conversions accumulate to "aabbb". + {"primconv_carryover", + `ec = Encoding::Converter.new("utf-8", "iso-8859-1") +dest = +"aa" +r1 = ec.primitive_convert(+"b", dest, nil, 0) +r2 = ec.primitive_convert(+"b", dest, nil, 1) +r3 = ec.primitive_convert(+"b", dest, nil, 2) +p [r1, r2, r3, dest]`, + "[:destination_buffer_full, :destination_buffer_full, :finished, \"aabbb\"]\n"}, + // A character whose encoding is larger than the destination cap returns + // :destination_buffer_full with the whole source consumed (the produced bytes + // are held), matching MRI. + {"primconv_buffer_full_clears_source", + `ec = Encoding::Converter.new("utf-8", "iso-2022-jp") +s = +"\u{9999}" +r = ec.primitive_convert(s, +"", 0, 2) +p [r, s]`, + "[:destination_buffer_full, \"\"]\n"}, + // UTF8-MAC (HFS+ NFD) is a valid Converter destination: a precomposed + // character is emitted in canonical decomposition (é -> "e" + combining acute), + // while an ASCII run is unchanged. + {"utf8mac_encode_nfd", + `ec = Encoding::Converter.new("UTF-8", "UTF8-MAC") +p ec.convert("é").bytes`, + "[101, 204, 129]\n"}, + {"utf8mac_encode_ascii", + `ec = Encoding::Converter.new("UTF-8", "UTF8-MAC") +p ec.convert("abc").bytes`, + "[97, 98, 99]\n"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { From ff01982e5afc1fc6e4e5f4b699b8ff33eb0788e4 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 6 Sep 2026 10:32:25 +0200 Subject: [PATCH 3/3] tests: cover carried>cap flush and xtext undefined-conversion branches Add primitive_convert carryover cases where the held output exceeds a later call's byte cap (exercising the room clamp and the produced>size hold), and a Shift_JIS undefined-conversion case exercising encodeCharTo's encoder-error path. All asserted against MRI Ruby 4.0.6. Co-Authored-By: Claude Opus 4.8 --- internal/vm/encoding_residuals_wave13_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/vm/encoding_residuals_wave13_test.go b/internal/vm/encoding_residuals_wave13_test.go index 56e6a042..16cf46aa 100644 --- a/internal/vm/encoding_residuals_wave13_test.go +++ b/internal/vm/encoding_residuals_wave13_test.go @@ -68,6 +68,17 @@ p ec.convert("é").bytes`, `ec = Encoding::Converter.new("UTF-8", "UTF8-MAC") p ec.convert("abc").bytes`, "[97, 98, 99]\n"}, + // When the output held from a previous call itself exceeds a later call's byte + // cap, only what fits is flushed and the rest stays held: a multi-byte + // character split across three capped calls still assembles the exact bytes. + {"primconv_carryover_exceeds_cap", + `ec = Encoding::Converter.new("utf-8", "utf-16be") +d = +"" +r1 = ec.primitive_convert(+"A", d, nil, 1) +r2 = ec.primitive_convert(+"", d, nil, 0) +r3 = ec.primitive_convert(+"", d, nil, 5) +p [r1, r2, r3, d.bytes]`, + "[:destination_buffer_full, :destination_buffer_full, :finished, [0, 65]]\n"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -83,6 +94,9 @@ p ec.convert("abc").bytes`, errCases := []struct{ name, src, want string }{ {"default_external_nil", `Encoding.default_external = nil`, "default external can not be nil"}, {"maccyrillic_undef", `"\u{6543}".encode("macCyrillic")`, "from UTF-8 to macCyrillic"}, + // A character an x/text-backed destination codec cannot represent drives the + // converter's undefined-conversion path (encodeCharTo's encoder-error branch). + {"sjis_undef", `Encoding::Converter.new("UTF-8", "Shift_JIS").convert("\u{1F600}")`, "UndefinedConversionError"}, } for _, tc := range errCases { t.Run(tc.name, func(t *testing.T) {