Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions internal/vm/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ import (
// codecs), driving them through a byte-at-a-time conversion loop so the
// #primitive_convert state machine (status symbols, source-buffer consumption,
// #primitive_errinfo, #last_error, #putback) matches MRI for the common single
// conversion step. The residual — MRI's internal output-buffer carryover across
// #primitive_convert calls (the "� spills into the next buffer" behaviour)
// and the multi-hop stateful decorators (ISO-2022-JP escape-state #finish) — is
// documented at the methods that would need it.
// conversion step. Output that does not fit a #primitive_convert destination is
// held in pendingOut and flushed on the next call (MRI's internal output-buffer
// carryover). The remaining residual — the multi-hop stateful decorators
// (ISO-2022-JP escape-state #finish) — is documented at the methods that would
// need it.
type converterObj struct {
src, dst string // canonical source / destination encoding names

Expand All @@ -41,6 +42,7 @@ type converterObj struct {
lastErrExc object.Value

pendingIncomplete []byte // incomplete tail buffered by #convert for #finish
pendingOut []byte // converted output that did not fit the destination, flushed on the next #primitive_convert
finished bool // #finish has been called (stream closed)
}

Expand Down
6 changes: 6 additions & 0 deletions internal/vm/converter_codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ package vm
import (
binpkg "encoding/binary"
"unicode/utf8"

"golang.org/x/text/unicode/norm"
)

// decodeCharFrom decodes the next character of in, interpreted in encoding
Expand Down Expand Up @@ -223,6 +225,10 @@ func (vm *VM) encodeCharTo(r rune, to string) ([]byte, bool) {
return encodeUTF32(string(r), false), true
case "UTF-32BE":
return encodeUTF32(string(r), true), true
case "UTF8-MAC":
// UTF8-MAC (a.k.a. UTF-8-MAC / HFS+) is UTF-8 in canonical decomposition
// (NFD), so each character is encoded as its NFD bytes.
return norm.NFD.Bytes([]byte(string(r))), true
}
enc, ok := xtextEncodings[to]
if !ok {
Expand Down
74 changes: 59 additions & 15 deletions internal/vm/converter_impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,8 @@ func (vm *VM) hopConvertible(from, to string) bool {
func encodingSupported(name string) bool {
switch name {
case "UTF-8", "US-ASCII", "ASCII-8BIT", "ISO-8859-1",
"UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE":
"UTF-16LE", "UTF-16BE", "UTF-32LE", "UTF-32BE",
"UTF8-MAC":
return true
}
_, ok := xtextEncodings[name]
Expand Down Expand Up @@ -324,7 +325,7 @@ func (vm *VM) converterConvert(c *converterObj, v object.Value) object.Value {
}
in := append(append([]byte(nil), c.readAgain...), vm.strToStr(v)...)
c.readAgain = nil
out, _, status, errBytes, readAgain := vm.stepLoop(c, in, -1)
out, _, status, errBytes, readAgain, _ := vm.stepLoop(c, in, -1)
switch status {
case "invalid_byte_sequence":
vm.setConvError(c, "invalid_byte_sequence", errBytes, readAgain, false)
Expand Down Expand Up @@ -406,7 +407,40 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu
}
}

out, consumed, status, errBytes, readAgain := vm.stepLoop(c, in, size)
// Output held from a previous destination_buffer_full is flushed first; only the
// remaining destination room is offered to the conversion of new source.
carried := c.pendingOut
c.pendingOut = nil
room := size
if size >= 0 {
room = size - len(carried)
if room < 0 {
room = 0
}
}

out, consumed, status, errBytes, readAgain, overflow := vm.stepLoop(c, in, room)

// Combine the carried output with this pass's output, then apply the destination
// cap: bytes past it (plus any stepLoop overflow) are held for the next call.
produced := append(append([]byte(nil), carried...), out...)
var hold []byte
if size >= 0 && len(produced) > size {
hold = append(hold, produced[size:]...)
produced = produced[:size]
}
hold = append(hold, overflow...)
if len(hold) > 0 {
c.pendingOut = hold
// Unwritten output means the destination is full — unless the pass stopped on
// a source error, which takes priority and keeps its bytes unconsumed.
switch status {
case "invalid_byte_sequence", "undefined_conversion", "incomplete_input":
default:
status = "destination_buffer_full"
}
}

if partial && status == "incomplete_input" {
status = "source_buffer_empty"
} else if !partial && status == "source_buffer_empty" {
Expand All @@ -419,7 +453,7 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu
if offset < len(base) {
base = base[:offset]
}
dst.SetBytes(append(append([]byte(nil), base...), out...))
dst.SetBytes(append(append([]byte(nil), base...), produced...))
dst.Enc = c.dst

// Consume the converted bytes from the source String (when one was given).
Expand All @@ -445,44 +479,53 @@ func (vm *VM) primitiveConvert(c *converterObj, args []object.Value) object.Valu

// stepLoop is the shared conversion engine for #convert and #primitive_convert.
// It decodes src character by character in the source encoding, encodes each into
// the destination encoding, and stops at the first error or when appending the
// next character would exceed size (a byte cap; negative means unlimited). It
// the destination encoding, and stops at the first error or when a character's
// encoded bytes no longer fit within size (a byte cap; negative means unlimited).
// When a character overflows the cap the bytes that fit go to out, the remainder
// to overflow, that character is consumed, and the loop stops with
// "destination_buffer_full" — the caller carries overflow to the next call. It
// returns the produced bytes, the number of source bytes consumed, a status
// string, and the erroneous / read-again bytes for the error statuses.
func (vm *VM) stepLoop(c *converterObj, in []byte, size int) (out []byte, consumed int, status string, errBytes, readAgain []byte) {
// string, the erroneous / read-again bytes for the error statuses, and the
// overflow bytes.
func (vm *VM) stepLoop(c *converterObj, in []byte, size int) (out []byte, consumed int, status string, errBytes, readAgain, overflow []byte) {
pos := 0
for pos < len(in) {
r, n, rl, st := vm.decodeCharFrom(in[pos:], c.src)
switch st {
case stepIncomplete:
if c.invalidReplace {
out = appendCapped(out, c.repl, size)
return out, len(in), "finished", nil, nil
return out, len(in), "finished", nil, nil, nil
}
return out, len(in), "incomplete_input", in[pos:], nil
return out, len(in), "incomplete_input", in[pos:], nil, nil
case stepInvalid:
if c.invalidReplace {
out = appendCapped(out, c.repl, size)
pos += n + rl
continue
}
return out, pos + n + rl, "invalid_byte_sequence", in[pos : pos+n], in[pos+n : pos+n+rl]
return out, pos + n + rl, "invalid_byte_sequence", in[pos : pos+n], in[pos+n : pos+n+rl], nil
}
eb, okEnc := vm.encodeCharTo(r, c.dst)
if !okEnc {
if c.undefReplace {
eb = c.repl
} else {
return out, pos + n, "undefined_conversion", []byte(string(r)), nil
return out, pos + n, "undefined_conversion", []byte(string(r)), nil, nil
}
}
if size >= 0 && len(out)+len(eb) > size {
return out, pos, "destination_buffer_full", nil, nil
// The character does not fit: write the prefix that fits, hold the rest,
// consume the character, and stop with the destination full.
room := size - len(out)
out = append(out, eb[:room]...)
overflow = append(overflow, eb[room:]...)
return out, pos + n, "destination_buffer_full", nil, nil, overflow
}
out = append(out, eb...)
pos += n
}
return out, len(in), "source_buffer_empty", nil, nil
return out, len(in), "source_buffer_empty", nil, nil, nil
}

// appendCapped appends add to out unless doing so would exceed a non-negative
Expand Down Expand Up @@ -570,7 +613,8 @@ func (vm *VM) converterPutback(c *converterObj, args []object.Value) object.Valu
b := c.readAgain[len(c.readAgain)-n:]
out := object.NewStringBytesEnc(append([]byte(nil), b...), c.src)
c.readAgain = c.readAgain[:len(c.readAgain)-n]
c.errReadAgain = c.readAgain
// #putback does not disturb #primitive_errinfo: MRI keeps reporting the
// original read-again bytes after a putback, so c.errReadAgain is left as set.
return out
}

Expand Down
30 changes: 25 additions & 5 deletions internal/vm/encoding.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,15 +125,21 @@ func (vm *VM) registerEncoding() {
// values Puppet's log_runtime_environment and IO setup read. The setters are
// accepted (and remembered) so code that brackets work in an encoding override
// runs; rbgo's string layer remains UTF-8 regardless.
defExternal := vm.encodings["UTF-8"]
vm.defExternalEnc = vm.encodings["UTF-8"]
sdef := func(name string, fn NativeFn) {
vm.cEncoding.smethods[name] = &Method{name: name, owner: vm.cEncoding, native: fn}
}
sdef("default_external", func(_ *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value {
return defExternal
sdef("default_external", func(vm *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value {
return vm.defExternalEnc
})
sdef("default_external=", func(vm *VM, _ object.Value, args []object.Value, _ *Proc) object.Value {
defExternal = vm.encodingArg(args[0])
// MRI rejects nil with ArgumentError (not the TypeError encodingArg would
// raise), and setting the default also redirects the "external",
// "filesystem" and "locale" names Encoding.find resolves.
if _, isNil := args[0].(object.Nil); isNil {
raise("ArgumentError", "default external can not be nil")
}
vm.defExternalEnc = vm.encodingArg(args[0])
return args[0]
})
sdef("default_internal", func(vm *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value {
Expand All @@ -157,8 +163,22 @@ func (vm *VM) registerEncoding() {
return object.NewString("UTF-8")
})
// Encoding.find(name) resolves a name (or a pass-through Encoding) to the
// registry object, raising ArgumentError for an unknown name as MRI does.
// registry object, raising ArgumentError for an unknown name as MRI does. The
// dynamic names track the current process defaults: "external"/"filesystem"/
// "locale" are Encoding.default_external, and "internal" is
// Encoding.default_internal (nil when unset).
sdef("find", func(vm *VM, _ object.Value, args []object.Value, _ *Proc) object.Value {
if s, ok := args[0].(*object.String); ok {
switch lower(s.Str()) {
case "external", "filesystem", "locale":
return vm.defExternalEnc
case "internal":
if vm.defInternalEnc == nil {
return object.NilV
}
return vm.defInternalEnc
}
}
return vm.encodingArg(args[0])
})
// Encoding.list returns every registered (non-alias) encoding, in registration
Expand Down
109 changes: 109 additions & 0 deletions internal/vm/encoding_residuals_wave13_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package vm_test

import (
"strings"
"testing"
)

// TestEncodingResidualsWave13 covers the encoding-subsystem conformance fixes of
// wave 13: the dynamic Encoding.find names ("external"/"filesystem"/"locale" ->
// default_external, "internal" -> default_internal), Encoding.default_external=
// rejecting nil, and the macCyrillic (Macintosh Cyrillic) transcoding codec.
// Every expectation is asserted against MRI Ruby 4.0.6.
func TestEncodingResidualsWave13(t *testing.T) {
cases := []struct{ name, src, want string }{
// Encoding.find resolves the dynamic external/filesystem/locale names to the
// current default_external, tracking the setter.
{"find_external_default", `p Encoding.find("external").name`, "\"UTF-8\"\n"},
{"find_locale_default", `p Encoding.find("locale").name`, "\"UTF-8\"\n"},
{"find_filesystem_default", `p Encoding.find("filesystem").name`, "\"UTF-8\"\n"},
{"find_external_tracks_setter",
`Encoding.default_external = Encoding::SHIFT_JIS
p [Encoding.find("external").name, Encoding.find("filesystem").name]`,
"[\"Shift_JIS\", \"Shift_JIS\"]\n"},
{"find_external_case_insensitive", `p Encoding.find("EXTERNAL").name`, "\"UTF-8\"\n"},
// Encoding.find("internal") is default_internal: nil when unset, the encoding
// once set.
{"find_internal_nil", `p Encoding.find("internal")`, "nil\n"},
{"find_internal_set",
`Encoding.default_internal = Encoding::US_ASCII
p Encoding.find("internal").name`,
"\"US-ASCII\"\n"},
// macCyrillic (Macintosh Cyrillic) transcodes via golang.org/x/text: a Cyrillic
// letter encodes to its single mac byte, an undefined character raises.
{"maccyrillic_encode", `p "А".encode("macCyrillic").bytes`, "[128]\n"},
{"maccyrillic_converter",
`ec = Encoding::Converter.new("UTF-8", Encoding.find("macCyrillic"))
p ec.convert("А").bytes`,
"[128]\n"},
// Encoding::Converter#primitive_convert carries output that did not fit the
// destination to the next call: successive calls with a growing byte cap flush
// the held byte before converting the next character (MRI's output-buffer
// carryover), so three one-byte conversions accumulate to "aabbb".
{"primconv_carryover",
`ec = Encoding::Converter.new("utf-8", "iso-8859-1")
dest = +"aa"
r1 = ec.primitive_convert(+"b", dest, nil, 0)
r2 = ec.primitive_convert(+"b", dest, nil, 1)
r3 = ec.primitive_convert(+"b", dest, nil, 2)
p [r1, r2, r3, dest]`,
"[:destination_buffer_full, :destination_buffer_full, :finished, \"aabbb\"]\n"},
// A character whose encoding is larger than the destination cap returns
// :destination_buffer_full with the whole source consumed (the produced bytes
// are held), matching MRI.
{"primconv_buffer_full_clears_source",
`ec = Encoding::Converter.new("utf-8", "iso-2022-jp")
s = +"\u{9999}"
r = ec.primitive_convert(s, +"", 0, 2)
p [r, s]`,
"[:destination_buffer_full, \"\"]\n"},
// UTF8-MAC (HFS+ NFD) is a valid Converter destination: a precomposed
// character is emitted in canonical decomposition (é -> "e" + combining acute),
// while an ASCII run is unchanged.
{"utf8mac_encode_nfd",
`ec = Encoding::Converter.new("UTF-8", "UTF8-MAC")
p ec.convert("é").bytes`,
"[101, 204, 129]\n"},
{"utf8mac_encode_ascii",
`ec = Encoding::Converter.new("UTF-8", "UTF8-MAC")
p ec.convert("abc").bytes`,
"[97, 98, 99]\n"},
// When the output held from a previous call itself exceeds a later call's byte
// cap, only what fits is flushed and the rest stays held: a multi-byte
// character split across three capped calls still assembles the exact bytes.
{"primconv_carryover_exceeds_cap",
`ec = Encoding::Converter.new("utf-8", "utf-16be")
d = +""
r1 = ec.primitive_convert(+"A", d, nil, 1)
r2 = ec.primitive_convert(+"", d, nil, 0)
r3 = ec.primitive_convert(+"", d, nil, 5)
p [r1, r2, r3, d.bytes]`,
"[:destination_buffer_full, :destination_buffer_full, :finished, [0, 65]]\n"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := eval(t, tc.src); got != tc.want {
t.Errorf("src=%q\n got=%q\nwant=%q", tc.src, got, tc.want)
}
})
}

// Error paths: default_external = nil raises ArgumentError (not the TypeError the
// generic coercion would give), and an undefined macCyrillic character raises
// Encoding::UndefinedConversionError.
errCases := []struct{ name, src, want string }{
{"default_external_nil", `Encoding.default_external = nil`, "default external can not be nil"},
{"maccyrillic_undef", `"\u{6543}".encode("macCyrillic")`, "from UTF-8 to macCyrillic"},
// A character an x/text-backed destination codec cannot represent drives the
// converter's undefined-conversion path (encodeCharTo's encoder-error branch).
{"sjis_undef", `Encoding::Converter.new("UTF-8", "Shift_JIS").convert("\u{1F600}")`, "UndefinedConversionError"},
}
for _, tc := range errCases {
t.Run(tc.name, func(t *testing.T) {
err := runErr(t, tc.src)
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Errorf("src=%q: got err=%v, want containing %q", tc.src, err, tc.want)
}
})
}
}
1 change: 1 addition & 0 deletions internal/vm/string_encode_xtext.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var xtextEncodings = map[string]encoding.Encoding{
"GB18030": simplifiedchinese.GB18030,
"Big5": traditionalchinese.Big5,
"macRoman": charmap.Macintosh,
"macCyrillic": charmap.MacintoshCyrillic,
}

// xtextDecode decodes src (in encoding `from`) to a UTF-8 string using x/text.
Expand Down
1 change: 1 addition & 0 deletions internal/vm/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,7 @@ type VM struct {
encodings map[string]*encodingObj // canonical name -> interned Encoding
encLookup map[string]*encodingObj // lowercased name/alias -> interned Encoding
defInternalEnc *encodingObj // Encoding.default_internal (nil = MRI's default)
defExternalEnc *encodingObj // Encoding.default_external (UTF-8 unless reset)
cLazy *RClass
lastMatch object.Value // $~: last regexp MatchData (or nil)
regexpTimeout object.Value // Regexp.timeout default (Float seconds or nil)
Expand Down