Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions internal/vm/encoding_encode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,52 @@ func TestStringEncode(t *testing.T) {
}
}
}

// TestStringEncodeFallback covers String#encode's :fallback option: a Hash, Proc
// or #[]-object supplies substitutes for characters unrepresentable in the target.
// Each value expectation was verified byte-for-byte against MRI Ruby 4.0.6.
func TestStringEncodeFallback(t *testing.T) {
cases := []struct{ src, want string }{
// Proc and Hash fallbacks; only the unrepresentable character is routed
// through, encodable ones pass straight to the encoder.
{`p "aéb".encode("US-ASCII", fallback: ->(c){ "?" })`, "\"a?b\"\n"},
{`p "aéb".encode("US-ASCII", fallback: {"é"=>"e"})`, "\"aeb\"\n"},
// The Proc receives the offending character as a String (c.ord == 233).
{`p "aéb".encode("US-ASCII", fallback: ->(c){ c.ord.to_s })`, "\"a233b\"\n"},
// undef: :replace pre-empts the fallback entirely (é becomes "?", not "e").
{`p "aéb".encode("US-ASCII", fallback: {"é"=>"e"}, undef: :replace)`, "\"a?b\"\n"},
// A Latin-1 target keeps representable characters (é, 0xE9) and only falls
// back for the truly absent one (€).
{`p "aé€b".encode("ISO-8859-1", fallback: {"€"=>"EUR"}).encode("utf-8")`, "\"aéEURb\"\n"},
// An x/text target (Shift_JIS) honours the fallback the same way.
{`p "a€b".encode("Shift_JIS", fallback: {"€"=>"E"}).bytes`, "[97, 69, 98]\n"},
// A target that can represent every character never consults the fallback.
{`p "aé".encode("UTF-16LE", fallback: {"x"=>"y"}).encode("utf-8")`, "\"aé\"\n"},
// The fallback Hash is keyed by the character in its own (UTF-8) encoding.
{`p "aéb".encode("US-ASCII", fallback: {"é".encode("UTF-8")=>"E"})`, "\"aEb\"\n"},
// Any object answering #[] serves as a fallback.
{`class EncFB; def [](c); "_"; end; end; p "aéb".encode("US-ASCII", fallback: EncFB.new)`, "\"a_b\"\n"},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want {
t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want)
}
}

errCases := []struct{ src, substr string }{
// A Hash miss or a nil Proc result leaves the character for the normal
// undefined-conversion error (MRI-verified).
{`"aéb".encode("US-ASCII", fallback: {"x"=>"y"})`, "UndefinedConversionError"},
{`"aéb".encode("US-ASCII", fallback: ->(c){ nil })`, "UndefinedConversionError"},
// A non-String, non-nil fallback result is a TypeError (MRI-verified).
{`"aéb".encode("US-ASCII", fallback: ->(c){ 42 })`, "no implicit conversion of Integer into String"},
// A target rbgo has no codec for is a named residual: the fallback cannot
// mask the missing converter (rbgo raises where MRI would transcode natively).
{`"aéb".encode("IBM437", fallback: {"é"=>"e"})`, "ConverterNotFoundError"},
}
for _, c := range errCases {
if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.substr) {
t.Errorf("src=%q err=%v, want containing %q", c.src, err, c.substr)
}
}
}
69 changes: 65 additions & 4 deletions internal/vm/string_encode.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ type transcodeOpts struct {
undefReplace bool // undef: :replace — substitute characters absent from the target
replace string // replace: "..." — the substitution string
hasReplace bool
crNewline bool // cr_newline: replace \n and \r\n with \r
crlfNewline bool // crlf_newline: replace \n with \r\n
universalNewline bool // universal_newline: replace \r\n and \r with \n
xml string // xml: :text or :attr — HTML/XML-escape the text
crNewline bool // cr_newline: replace \n and \r\n with \r
crlfNewline bool // crlf_newline: replace \n with \r\n
universalNewline bool // universal_newline: replace \r\n and \r with \n
xml string // xml: :text or :attr — HTML/XML-escape the text
fallback object.Value // fallback: a Hash/Proc/Method/#[]-object mapping an unrepresentable character to its substitute
hasFallback bool
}

// isUnicodeEncoding reports whether name is one of Ruby's Unicode encodings, for
Expand Down Expand Up @@ -75,6 +77,9 @@ func parseEncodeOpts(h *object.Hash) transcodeOpts {
raise("ArgumentError", "unexpected value for xml option: %s", v.Inspect())
}
}
if v, ok := sym("fallback"); ok && truthyValue(v) {
o.fallback, o.hasFallback = v, true
}
return o
}

Expand All @@ -97,6 +102,56 @@ func applyXML(s string, o transcodeOpts) string {

func truthyValue(v object.Value) bool { return v != object.NilV && v != object.Bool(false) }

// charEncodableIn reports whether rune r can be represented in encoding `to`. The
// :fallback pre-pass uses it to route only the genuinely unrepresentable
// characters through the fallback, leaving the rest for the normal encoder.
func charEncodableIn(r rune, to string) bool {
switch to {
case "UTF-8", "UTF-16", "UTF-16LE", "UTF-16BE", "UTF-32", "UTF-32LE", "UTF-32BE":
return true
case "US-ASCII", "ASCII-8BIT":
return r < 0x80
case "ISO-8859-1":
return r < 0x100
}
if enc, ok := xtextEncodings[to]; ok {
_, err := enc.NewEncoder().Bytes([]byte(string(r)))
return err == nil
}
// An unknown target has no codec; leave the character in place so the encoder
// raises Encoding::ConverterNotFoundError rather than the fallback masking it.
return true
}

// applyFallback rewrites the UTF-8 intermediate u, replacing each character that
// cannot be represented in the target `to` with the result of the :fallback
// option — a Hash, Proc, Method or any object queried with #[], passed the
// character as a String. A nil result (a Hash miss included) leaves the character
// in place for the normal undefined-conversion error; a non-String, non-nil
// result raises TypeError, matching MRI. It is called only when undef: :replace is
// off (that option pre-empts the fallback entirely).
func (vm *VM) applyFallback(u, to string, fb object.Value) string {
var b strings.Builder
for _, r := range u {
if charEncodableIn(r, to) {
b.WriteRune(r)
continue
}
res := vm.send(fb, "[]", []object.Value{object.NewString(string(r))}, nil)
switch v := res.(type) {
case *object.String:
b.WriteString(v.Str())
default:
if res == object.NilV {
b.WriteRune(r)
continue
}
raise("TypeError", "no implicit conversion of %s into String", vm.classOf(res).name)
}
}
return b.String()
}

// applyNewline rewrites newline sequences in the UTF-8 intermediate per the
// cr_newline / crlf_newline / universal_newline options.
func applyNewline(s string, o transcodeOpts) string {
Expand Down Expand Up @@ -335,6 +390,12 @@ func (vm *VM) stringEncode(s *object.String, args []object.Value) *object.String
inter := vm.decodeToUTF8(s.Bytes(), from, opts, repl)
inter = applyNewline(inter, opts)
inter = applyXML(inter, opts)
// The :fallback option supplies substitutes for unrepresentable characters,
// but only where undef: :replace does not already handle them — that option
// pre-empts the fallback (MRI never consults it once undef substitution is on).
if opts.hasFallback && !opts.undefReplace {
inter = vm.applyFallback(inter, to, opts.fallback)
}
out := vm.encodeFromUTF8(inter, to, opts, repl)
return object.NewStringBytesEnc(out, to)
}
Expand Down
2 changes: 1 addition & 1 deletion scripts/conformance/rubyspec/FLOOR
Original file line number Diff line number Diff line change
@@ -1 +1 @@
16290
16305