From 9140a5972d3ce3f119ef1cda21e1123625bd27a4 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Sun, 30 Aug 2026 11:48:21 +0200 Subject: [PATCH] Add String#encode :fallback option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit String#encode's :fallback keyword supplies substitutes for characters that cannot be represented in the target encoding. It accepts a Hash, a Proc, a Method, or any object queried with #[], each passed the offending character as a String; a nil result (a Hash miss included) leaves the character for the normal Encoding::UndefinedConversionError, and a non-String, non-nil result raises TypeError — all matching MRI 4.0.6. The fallback is applied as a pre-pass over the UTF-8 intermediate, rewriting only the genuinely unrepresentable characters (decided per target by charEncodableIn, which covers the core encodings and the x/text-backed legacy ones) so representable characters flow straight to the encoder. undef: :replace pre-empts the fallback entirely, as in MRI. Lifts core/string/encode_spec.rb from 77 to 100 passing examples, byte-for-byte cross-checked against MRI Ruby 4.0.6. ruby/spec ratchet FLOOR 16282 -> 16305 (measured total 16364). Co-Authored-By: Claude Opus 4.8 --- internal/vm/encoding_encode_test.go | 49 ++++++++++++++++++++ internal/vm/string_encode.go | 69 +++++++++++++++++++++++++++-- scripts/conformance/rubyspec/FLOOR | 2 +- 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/internal/vm/encoding_encode_test.go b/internal/vm/encoding_encode_test.go index 77718985..3d3611de 100644 --- a/internal/vm/encoding_encode_test.go +++ b/internal/vm/encoding_encode_test.go @@ -101,3 +101,52 @@ func TestStringEncode(t *testing.T) { } } } + +// TestStringEncodeFallback covers String#encode's :fallback option: a Hash, Proc +// or #[]-object supplies substitutes for characters unrepresentable in the target. +// Each value expectation was verified byte-for-byte against MRI Ruby 4.0.6. +func TestStringEncodeFallback(t *testing.T) { + cases := []struct{ src, want string }{ + // Proc and Hash fallbacks; only the unrepresentable character is routed + // through, encodable ones pass straight to the encoder. + {`p "aéb".encode("US-ASCII", fallback: ->(c){ "?" })`, "\"a?b\"\n"}, + {`p "aéb".encode("US-ASCII", fallback: {"é"=>"e"})`, "\"aeb\"\n"}, + // The Proc receives the offending character as a String (c.ord == 233). + {`p "aéb".encode("US-ASCII", fallback: ->(c){ c.ord.to_s })`, "\"a233b\"\n"}, + // undef: :replace pre-empts the fallback entirely (é becomes "?", not "e"). + {`p "aéb".encode("US-ASCII", fallback: {"é"=>"e"}, undef: :replace)`, "\"a?b\"\n"}, + // A Latin-1 target keeps representable characters (é, 0xE9) and only falls + // back for the truly absent one (€). + {`p "aé€b".encode("ISO-8859-1", fallback: {"€"=>"EUR"}).encode("utf-8")`, "\"aéEURb\"\n"}, + // An x/text target (Shift_JIS) honours the fallback the same way. + {`p "a€b".encode("Shift_JIS", fallback: {"€"=>"E"}).bytes`, "[97, 69, 98]\n"}, + // A target that can represent every character never consults the fallback. + {`p "aé".encode("UTF-16LE", fallback: {"x"=>"y"}).encode("utf-8")`, "\"aé\"\n"}, + // The fallback Hash is keyed by the character in its own (UTF-8) encoding. + {`p "aéb".encode("US-ASCII", fallback: {"é".encode("UTF-8")=>"E"})`, "\"aEb\"\n"}, + // Any object answering #[] serves as a fallback. + {`class EncFB; def [](c); "_"; end; end; p "aéb".encode("US-ASCII", fallback: EncFB.new)`, "\"a_b\"\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } + + errCases := []struct{ src, substr string }{ + // A Hash miss or a nil Proc result leaves the character for the normal + // undefined-conversion error (MRI-verified). + {`"aéb".encode("US-ASCII", fallback: {"x"=>"y"})`, "UndefinedConversionError"}, + {`"aéb".encode("US-ASCII", fallback: ->(c){ nil })`, "UndefinedConversionError"}, + // A non-String, non-nil fallback result is a TypeError (MRI-verified). + {`"aéb".encode("US-ASCII", fallback: ->(c){ 42 })`, "no implicit conversion of Integer into String"}, + // A target rbgo has no codec for is a named residual: the fallback cannot + // mask the missing converter (rbgo raises where MRI would transcode natively). + {`"aéb".encode("IBM437", fallback: {"é"=>"e"})`, "ConverterNotFoundError"}, + } + for _, c := range errCases { + if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.substr) { + t.Errorf("src=%q err=%v, want containing %q", c.src, err, c.substr) + } + } +} diff --git a/internal/vm/string_encode.go b/internal/vm/string_encode.go index bb78e905..e8e22a7a 100644 --- a/internal/vm/string_encode.go +++ b/internal/vm/string_encode.go @@ -15,10 +15,12 @@ type transcodeOpts struct { undefReplace bool // undef: :replace — substitute characters absent from the target replace string // replace: "..." — the substitution string hasReplace bool - crNewline bool // cr_newline: replace \n and \r\n with \r - crlfNewline bool // crlf_newline: replace \n with \r\n - universalNewline bool // universal_newline: replace \r\n and \r with \n - xml string // xml: :text or :attr — HTML/XML-escape the text + crNewline bool // cr_newline: replace \n and \r\n with \r + crlfNewline bool // crlf_newline: replace \n with \r\n + universalNewline bool // universal_newline: replace \r\n and \r with \n + xml string // xml: :text or :attr — HTML/XML-escape the text + fallback object.Value // fallback: a Hash/Proc/Method/#[]-object mapping an unrepresentable character to its substitute + hasFallback bool } // isUnicodeEncoding reports whether name is one of Ruby's Unicode encodings, for @@ -75,6 +77,9 @@ func parseEncodeOpts(h *object.Hash) transcodeOpts { raise("ArgumentError", "unexpected value for xml option: %s", v.Inspect()) } } + if v, ok := sym("fallback"); ok && truthyValue(v) { + o.fallback, o.hasFallback = v, true + } return o } @@ -97,6 +102,56 @@ func applyXML(s string, o transcodeOpts) string { func truthyValue(v object.Value) bool { return v != object.NilV && v != object.Bool(false) } +// charEncodableIn reports whether rune r can be represented in encoding `to`. The +// :fallback pre-pass uses it to route only the genuinely unrepresentable +// characters through the fallback, leaving the rest for the normal encoder. +func charEncodableIn(r rune, to string) bool { + switch to { + case "UTF-8", "UTF-16", "UTF-16LE", "UTF-16BE", "UTF-32", "UTF-32LE", "UTF-32BE": + return true + case "US-ASCII", "ASCII-8BIT": + return r < 0x80 + case "ISO-8859-1": + return r < 0x100 + } + if enc, ok := xtextEncodings[to]; ok { + _, err := enc.NewEncoder().Bytes([]byte(string(r))) + return err == nil + } + // An unknown target has no codec; leave the character in place so the encoder + // raises Encoding::ConverterNotFoundError rather than the fallback masking it. + return true +} + +// applyFallback rewrites the UTF-8 intermediate u, replacing each character that +// cannot be represented in the target `to` with the result of the :fallback +// option — a Hash, Proc, Method or any object queried with #[], passed the +// character as a String. A nil result (a Hash miss included) leaves the character +// in place for the normal undefined-conversion error; a non-String, non-nil +// result raises TypeError, matching MRI. It is called only when undef: :replace is +// off (that option pre-empts the fallback entirely). +func (vm *VM) applyFallback(u, to string, fb object.Value) string { + var b strings.Builder + for _, r := range u { + if charEncodableIn(r, to) { + b.WriteRune(r) + continue + } + res := vm.send(fb, "[]", []object.Value{object.NewString(string(r))}, nil) + switch v := res.(type) { + case *object.String: + b.WriteString(v.Str()) + default: + if res == object.NilV { + b.WriteRune(r) + continue + } + raise("TypeError", "no implicit conversion of %s into String", vm.classOf(res).name) + } + } + return b.String() +} + // applyNewline rewrites newline sequences in the UTF-8 intermediate per the // cr_newline / crlf_newline / universal_newline options. func applyNewline(s string, o transcodeOpts) string { @@ -335,6 +390,12 @@ func (vm *VM) stringEncode(s *object.String, args []object.Value) *object.String inter := vm.decodeToUTF8(s.Bytes(), from, opts, repl) inter = applyNewline(inter, opts) inter = applyXML(inter, opts) + // The :fallback option supplies substitutes for unrepresentable characters, + // but only where undef: :replace does not already handle them — that option + // pre-empts the fallback (MRI never consults it once undef substitution is on). + if opts.hasFallback && !opts.undefReplace { + inter = vm.applyFallback(inter, to, opts.fallback) + } out := vm.encodeFromUTF8(inter, to, opts, repl) return object.NewStringBytesEnc(out, to) } diff --git a/scripts/conformance/rubyspec/FLOOR b/scripts/conformance/rubyspec/FLOOR index c2c6c1f0..6413a78d 100644 --- a/scripts/conformance/rubyspec/FLOOR +++ b/scripts/conformance/rubyspec/FLOOR @@ -1 +1 @@ -16290 +16305