Skip to content

Commit 8a510ae

Browse files
tannevaledclaude
andauthored
Advance Marshal conformance: Data dump/load and Regexp encoding (#524)
* Marshal: dump and load Data as an 'S' struct container Data.define instances (Ruby 3.2+ immutable value objects) store their members in RObject.structVals the same way a Struct does, and MRI marshals them with the same 'S' container. Emit that on dump and accept it on load via memberDefNames, reconstructing a frozen Data instance. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * Marshal: encode a Regexp's encoding and FIXED/NONE option bits MRI records a Regexp's encoding the same way it records a String's: bare (no encoding ivar) for a binary source, an :E => true/false marker for UTF-8/US-ASCII, and a named :encoding ivar otherwise. The option byte carries the FIXEDENCODING (0x10) and NOENCODING (0x20) bits, and an ASCII-incompatible source (UTF-16/UTF-32) is fixed even when its bytes are ASCII-only. Compute the encoding from the compiled Regexp (via encodingName plus the fixed source encoding), pick the option byte to match, and restore both on load so a Regexp round-trips byte-for-byte. A non-ASCII source stays content-fixed (its fixedEnc field left false) so a loaded /café/ still equals the literal. Fixes the four encoding examples in core/marshal/dump_spec (binary, ascii-compatible, UTF-8, another-encoding); the remaining Regexp residuals need the object model to carry a Regexp subclass identity and ivars. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 5940453 commit 8a510ae

2 files changed

Lines changed: 202 additions & 11 deletions

File tree

‎internal/vm/marshal_codec.go‎

Lines changed: 112 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -253,8 +253,49 @@ func (d *mDumper) writeHash(h *object.Hash) {
253253
}
254254
}
255255

256+
// encodingASCIICompatible reports whether an encoding name denotes an
257+
// ASCII-compatible encoding — one whose ASCII bytes stand for ASCII characters.
258+
// The wide Unicode transformation formats (UTF-16/UTF-32, in any endianness) are
259+
// the exceptions Marshal cares about: a Regexp over such a source keeps that
260+
// encoding even when every byte happens to be ASCII.
261+
func encodingASCIICompatible(name string) bool {
262+
u := strings.ToUpper(name)
263+
return !strings.HasPrefix(u, "UTF-16") && !strings.HasPrefix(u, "UTF-32")
264+
}
265+
266+
// regexpMarshalFixed reports whether the Regexp is dumped with the
267+
// ARG_ENCODING_FIXED (0x10) option bit set. MRI sets it when FIXEDENCODING was
268+
// requested or the source is otherwise tied to a concrete encoding: a byte
269+
// outside ASCII, or an ASCII-incompatible source encoding (UTF-16/UTF-32) even
270+
// with ASCII-only bytes. A NOENCODING (/n) Regexp is not fixed.
271+
func regexpMarshalFixed(r *Regexp) bool {
272+
if r.fixedEnc {
273+
return true
274+
}
275+
if r.noEnc {
276+
return false
277+
}
278+
if r.srcEnc != "" && !encodingASCIICompatible(r.srcEnc) {
279+
return true
280+
}
281+
return !marshalIsASCII(r.source)
282+
}
283+
284+
// regexpMarshalEnc returns the encoding name Marshal records for the Regexp,
285+
// which is the encoding of the compiled Regexp (#encoding). When the Regexp is
286+
// fixed to a concrete source encoding, that encoding is recorded even for an
287+
// ASCII-only source (so a Windows-1251 or UTF-16 Regexp keeps its encoding);
288+
// otherwise #encoding's own rule applies (US-ASCII for an ASCII-only source, the
289+
// source encoding for a binary or wide one). An "ASCII-8BIT" result is dumped
290+
// bare, with no encoding ivar.
291+
func regexpMarshalEnc(r *Regexp) string {
292+
if regexpMarshalFixed(r) && r.srcEnc != "" {
293+
return r.srcEnc
294+
}
295+
return r.encodingName()
296+
}
297+
256298
func (d *mDumper) writeRegexp(r *Regexp) {
257-
nonASCII := !marshalIsASCII(r.source)
258299
opt := 0
259300
if strings.ContainsRune(r.flags, 'i') {
260301
opt |= reIgnoreCase
@@ -265,18 +306,35 @@ func (d *mDumper) writeRegexp(r *Regexp) {
265306
if strings.ContainsRune(r.flags, 'm') {
266307
opt |= reMultiline
267308
}
268-
if nonASCII {
269-
opt |= 16 // ARG_ENCODING_FIXED, set for a non-ASCII (UTF-8) source
309+
if regexpMarshalFixed(r) {
310+
opt |= reFixedEncoding
311+
}
312+
if r.noEnc {
313+
opt |= reNoEncoding
314+
}
315+
enc := regexpMarshalEnc(r)
316+
if enc == "ASCII-8BIT" {
317+
// A binary (ASCII-8BIT) Regexp carries no encoding ivar, so it is emitted
318+
// bare — no 'I' wrapper.
319+
d.buf = append(d.buf, '/')
320+
d.writeBytes(r.source)
321+
d.buf = append(d.buf, byte(opt))
322+
return
270323
}
271324
d.buf = append(d.buf, 'I', '/')
272325
d.writeBytes(r.source)
273326
d.buf = append(d.buf, byte(opt))
274327
d.writeLong(1)
275-
d.writeSymbol("E")
276-
if nonASCII {
328+
switch enc {
329+
case "UTF-8":
330+
d.writeSymbol("E")
277331
d.buf = append(d.buf, 'T')
278-
} else {
332+
case "US-ASCII":
333+
d.writeSymbol("E")
279334
d.buf = append(d.buf, 'F')
335+
default:
336+
d.writeSymbol("encoding")
337+
d.writeValue(object.NewStringBytesEnc([]byte(enc), "ASCII-8BIT"))
280338
}
281339
}
282340

@@ -386,6 +444,20 @@ func (d *mDumper) writeObject(o *RObject) {
386444
}
387445
return
388446
}
447+
// A Data (Ruby 3.2+ immutable value object, minted by Data.define) marshals
448+
// with the same 'S' container as a Struct — the class name followed by each
449+
// member name/value pair — because MRI stores a Data's members the same way it
450+
// stores a Struct's. The real class name is used (never a #name override).
451+
if dd := dataDefOf(o.class); dd != nil {
452+
d.buf = append(d.buf, 'S')
453+
d.writeSymbol(o.class.name)
454+
d.writeLong(len(dd.names))
455+
for i, name := range dd.names {
456+
d.writeSymbol(name)
457+
d.writeValue(o.structVals[i])
458+
}
459+
return
460+
}
389461
// An instance of a user subclass of a built-in value type (Array/Hash/
390462
// String/Range/...) dumps as 'C' — the class name followed by the wrapped
391463
// built-in value. Any instance variables are carried by an outer 'I' wrapper,
@@ -792,7 +864,20 @@ func (r *mReader) applyIvar(base object.Value, name string, val object.Value) {
792864
}
793865
}
794866
case *Regexp:
795-
// :E only refines the source encoding; the compiled Regexp needs nothing.
867+
// The encoding ivar records the Regexp's source encoding, which #encoding
868+
// reports (and a re-dump reproduces); the compiled matcher is unaffected.
869+
switch name {
870+
case "E":
871+
if val.Truthy() {
872+
b.srcEnc = "UTF-8"
873+
} else {
874+
b.srcEnc = "US-ASCII"
875+
}
876+
case "encoding":
877+
if s, ok := val.(*object.String); ok {
878+
b.srcEnc = s.Str()
879+
}
880+
}
796881
default:
797882
setIvar(base, name, val)
798883
}
@@ -899,22 +984,30 @@ func (r *mReader) readRange() object.Value {
899984
func (r *mReader) readStruct() object.Value {
900985
className := r.readSymbol()
901986
cls := r.vm.marshalClass(className)
902-
sd := structDefOf(cls)
903-
if sd == nil {
987+
// The 'S' container carries both Struct and Data (Ruby 3.2+ immutable value)
988+
// instances, whose members are stored the same way; memberDefNames resolves
989+
// either layout. A Data loads as a frozen instance, matching MRI's immutability
990+
// (independent of the freeze: kwarg).
991+
names, ok := memberDefNames(cls)
992+
if !ok {
904993
raise("TypeError", "%s is not a Struct", className)
905994
}
906-
o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(sd.names))}
995+
isData := dataDefOf(cls) != nil
996+
o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(names))}
907997
r.register(o)
908998
n := r.long()
909999
for i := 0; i < n; i++ {
9101000
name := r.readSymbol()
9111001
val := r.readValue()
912-
for idx, m := range sd.names {
1002+
for idx, m := range names {
9131003
if m == name {
9141004
o.structVals[idx] = val
9151005
}
9161006
}
9171007
}
1008+
if isData {
1009+
o.frozen = true
1010+
}
9181011
return o
9191012
}
9201013

@@ -992,6 +1085,14 @@ func (r *mReader) readRegexp(_ bool) object.Value {
9921085
flags += "x"
9931086
}
9941087
re := r.vm.compileRegexp(source, flags)
1088+
if rx, ok := re.(*Regexp); ok {
1089+
// A non-ASCII source is already treated as fixed-encoding by its content,
1090+
// exactly as a source literal is, so recording the FIXED bit for it would
1091+
// make a loaded /café/ unequal to the literal /café/ (whose fixedEnc field
1092+
// is false). Only an ASCII-only source needs the bit carried explicitly.
1093+
rx.fixedEnc = opt&reFixedEncoding != 0 && marshalIsASCII(source)
1094+
rx.noEnc = opt&reNoEncoding != 0
1095+
}
9951096
return r.register(re)
9961097
}
9971098

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package vm_test
2+
3+
import (
4+
"strings"
5+
"testing"
6+
)
7+
8+
// TestMarshalResiduals covers the Marshal conformance residuals fixed on top of
9+
// the earlier passes: Data (Ruby 3.2+ immutable value objects) dump/load via the
10+
// 'S' container, and the Regexp encoding model (the option byte's
11+
// FIXEDENCODING / NOENCODING bits and the recorded source encoding). Every
12+
// byte-exact expectation was captured from MRI Ruby 4.0
13+
// (`ruby -e 'p Marshal.dump(x).bytes'`).
14+
func TestMarshalResiduals(t *testing.T) {
15+
cases := []struct{ src, want string }{
16+
// --- Data: 'S' container, member name/value pairs -----------------
17+
{`M = Data.define(:amount, :unit); p Marshal.dump(M.new(100, "km".b)).bytes`,
18+
"[4, 8, 83, 58, 6, 77, 7, 58, 11, 97, 109, 111, 117, 110, 116, 105, 105, 58, 9, 117, 110, 105, 116, 34, 7, 107, 109]\n"},
19+
// A Data round-trips to a frozen instance with its members restored.
20+
{`MB = Data.define(:a, :b); v = Marshal.load(Marshal.dump(MB.new(1, 2))); p [v.a, v.b]`, "[1, 2]\n"},
21+
{`MF = Data.define(:a); p Marshal.load(Marshal.dump(MF.new(1))).frozen?`, "true\n"},
22+
// A plain subclass of a Data.define class shares the parent's members.
23+
{`MP = Data.define(:a); class MPSub < MP; end; v = Marshal.load(Marshal.dump(MPSub.new(9))); p [v.class.name, v.a]`,
24+
"[\"MPSub\", 9]\n"},
25+
// The real class name is used, never a #name override.
26+
{`MN = Data.define(:a); def MN.name; "Nope"; end; d = Marshal.dump(MN.new(1)); p [d.include?("MN"), d.include?("Nope")]`, "[true, false]\n"},
27+
28+
// --- Regexp: encoding + option byte -------------------------------
29+
// Binary (ASCII-8BIT) source: dumped bare, no encoding ivar, FIXED bit set.
30+
{`p Marshal.dump(Regexp.new("".b, Regexp::FIXEDENCODING)).bytes`, "[4, 8, 47, 0, 16]\n"},
31+
// US-ASCII source, FIXEDENCODING: E => false, FIXED bit set.
32+
{`p Marshal.dump(Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING)).bytes`,
33+
"[4, 8, 73, 47, 6, 97, 16, 6, 58, 6, 69, 70]\n"},
34+
// US-ASCII source, not fixed: E => false, no FIXED bit.
35+
{`p Marshal.dump(Regexp.new("a".encode("us-ascii"))).bytes`,
36+
"[4, 8, 73, 47, 6, 97, 0, 6, 58, 6, 69, 70]\n"},
37+
// Windows-1251 (ASCII-compatible) source, FIXEDENCODING: named :encoding ivar.
38+
{`p Marshal.dump(Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING)).bytes`,
39+
"[4, 8, 73, 47, 6, 97, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 17, 87, 105, 110, 100, 111, 119, 115, 45, 49, 50, 53, 49]\n"},
40+
// UTF-8 source, FIXEDENCODING: E => true.
41+
{`p Marshal.dump(Regexp.new("".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING)).bytes`,
42+
"[4, 8, 73, 47, 0, 16, 6, 58, 6, 69, 84]\n"},
43+
// UTF-16LE (ASCII-incompatible) source: :encoding ivar, FIXED bit.
44+
{`p Marshal.dump(Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING)).bytes`,
45+
"[4, 8, 73, 47, 7, 97, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 49, 54, 76, 69]\n"},
46+
// UTF-32LE source, NOT fixed by the caller: an ASCII-incompatible source
47+
// is fixed anyway, so FIXED is set and the encoding is recorded.
48+
{`p Marshal.dump(Regexp.new("a".encode("utf-32le"))).bytes`,
49+
"[4, 8, 73, 47, 9, 97, 0, 0, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 51, 50, 76, 69]\n"},
50+
// A UTF-8 literal with a non-ASCII byte is implicitly fixed: FIXED bit, E => true.
51+
{`p Marshal.dump(Regexp.new("café".dup.force_encoding("utf-8"))).bytes`,
52+
"[4, 8, 73, 47, 10, 99, 97, 102, 195, 169, 16, 6, 58, 6, 69, 84]\n"},
53+
// An ASCII-only literal: US-ASCII, no FIXED bit.
54+
{`p Marshal.dump(/\A.\Z/).bytes`, "[4, 8, 73, 47, 10, 92, 65, 46, 92, 90, 0, 6, 58, 6, 69, 70]\n"},
55+
// Flag letters map to the low option bits (i|m = 5); still US-ASCII.
56+
{`p Marshal.dump(//im).bytes`, "[4, 8, 73, 47, 0, 5, 6, 58, 6, 69, 70]\n"},
57+
// NOENCODING over an ASCII-only source: encoding stays US-ASCII (E => false),
58+
// only the NONE option bit (32) is set — not FIXED.
59+
{`p Marshal.dump(Regexp.new("a", Regexp::NOENCODING)).bytes`,
60+
"[4, 8, 73, 47, 6, 97, 32, 6, 58, 6, 69, 70]\n"},
61+
62+
// --- Regexp reader: options + encoding survive a round-trip -------
63+
// dump -> load -> dump is byte-stable for each encoding shape, which
64+
// exercises the reader restoring the FIXED / NONE option bits and the
65+
// source encoding from the E / :encoding ivar.
66+
{`re = Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
67+
{`re = Regexp.new("café".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
68+
{`re = Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
69+
{`re = Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
70+
{`re = Regexp.new("a".encode("utf-32le")); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
71+
{`re = Regexp.new("a", Regexp::NOENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
72+
{`re = /\A.\Z/; p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
73+
}
74+
for _, c := range cases {
75+
if got := eval(t, c.src); got != c.want {
76+
t.Errorf("src=%q got=%q want=%q", c.src, got, c.want)
77+
}
78+
}
79+
80+
errs := []struct{ src, want string }{
81+
// An 'S' container naming a class with no member layout is rejected, the
82+
// same as it is for a non-Struct (here a plain object class).
83+
{`class NotAData; end; Marshal.load("\x04\bS:\rNotAData\x00")`, "is not a Struct"},
84+
}
85+
for _, c := range errs {
86+
if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.want) {
87+
t.Errorf("src=%q got=%v want error containing %q", c.src, err, c.want)
88+
}
89+
}
90+
}

0 commit comments

Comments
 (0)