diff --git a/internal/vm/builtins.go b/internal/vm/builtins.go index 682bd7df..75bd41d6 100644 --- a/internal/vm/builtins.go +++ b/internal/vm/builtins.go @@ -1465,12 +1465,21 @@ func (vm *VM) bootstrap() { vm.cString.define("empty?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return object.Bool(len(strOf(self)) == 0) }) - vm.cString.define("dump", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { - // Always a plain (never frozen, never subclass) String in the receiver's - // encoding; a non-ASCII-compatible source additionally carries a - // .force_encoding suffix inside the dumped text (see (*String).Dump). + vm.cString.define("dump", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + // Always a plain (never frozen, never subclass) String. The escaped text is + // pure ASCII, so it keeps the receiver's encoding when that is ASCII- + // compatible, but a non-ASCII-compatible source degrades to US-ASCII and + // carries a .force_encoding suffix inside the dumped text (see + // (*String).Dump) — this is what lets String#undump read it back. s := self.(*object.String) - return object.NewStringBytesEnc([]byte(s.Dump()), s.EncName()) + enc := s.EncName() + if e, ok := vm.findEncoding(enc); ok && !e.asciiCompat { + enc = "US-ASCII" + } + return object.NewStringBytesEnc([]byte(s.Dump()), enc) + }) + vm.cString.define("undump", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return vm.stringUndump(self.(*object.String)) }) vm.cString.define("upcase", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return strEncOf(self, strings.ToUpper(strOf(self))) diff --git a/internal/vm/string_undump.go b/internal/vm/string_undump.go new file mode 100644 index 00000000..d0527aa0 --- /dev/null +++ b/internal/vm/string_undump.go @@ -0,0 +1,189 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import ( + "strings" + "unicode/utf8" + + "github.com/go-embedded-ruby/ruby/internal/object" +) + +// undumpNamedEscape maps the backslash-escaped control letters String#undump +// recognizes back to their byte value (the inverse of Dump's named escapes). +var undumpNamedEscape = map[byte]byte{ + 'a': 0x07, 'b': 0x08, 't': 0x09, 'n': 0x0a, + 'v': 0x0b, 'f': 0x0c, 'r': 0x0d, 'e': 0x1b, + '"': '"', '\\': '\\', '#': '#', +} + +// undumpHexNibble returns the value of a single hexadecimal digit and whether c is one. +func undumpHexNibble(c byte) (int, bool) { + switch { + case c >= '0' && c <= '9': + return int(c - '0'), true + case c >= 'a' && c <= 'f': + return int(c-'a') + 10, true + case c >= 'A' && c <= 'F': + return int(c-'A') + 10, true + } + return 0, false +} + +// stringUndump implements String#undump: it parses a String#dump literal back +// into the original string. The receiver must be an ASCII-compatible encoding +// (else Encoding::CompatibilityError); the text must be wrapped in double quotes, +// optionally followed by .force_encoding("NAME"); every backslash escape is +// decoded; and a raw NUL or non-ASCII byte, a malformed escape, or trailing junk +// raises RuntimeError. Verified against ruby 4.0.6. +func (vm *VM) stringUndump(s *object.String) object.Value { + if e, ok := vm.findEncoding(s.EncName()); ok && !e.asciiCompat { + raise("Encoding::CompatibilityError", "ASCII incompatible encoding: %s", s.EncName()) + } + data := s.Bytes() + n := len(data) + invalid := func() { + raise("RuntimeError", `invalid dumped string; not wrapped with '"' nor '"...".force_encoding("...")' form`) + } + if n == 0 || data[0] != '"' { + invalid() + } + + var out []byte + i := 1 + closed := false + for i < n { + c := data[i] + if c == '"' { + i++ + closed = true + break + } + if c == 0 { + raise("RuntimeError", "string contains null byte") + } + if c >= 0x80 { + raise("RuntimeError", "non-ASCII character detected") + } + if c != '\\' { + out = append(out, c) + i++ + continue + } + // A backslash escape. + i++ + if i >= n { + raise("RuntimeError", "invalid escape") + } + esc := data[i] + if b, ok := undumpNamedEscape[esc]; ok { + out = append(out, b) + i++ + continue + } + switch esc { + case 'x': + i++ + hi, ok1 := hexAt(data, i) + lo, ok2 := hexAt(data, i+1) + if !ok1 || !ok2 { + raise("RuntimeError", "invalid hex escape") + } + out = append(out, byte(hi<<4|lo)) + i += 2 + case 'u': + i++ + out = vm.undumpUnicode(data, &i, out) + default: + // dump never emits any other escape; MRI leaves an unrecognized one + // as-is, keeping the backslash (\q undumps to \q). + out = append(out, '\\', esc) + i++ + } + } + if !closed { + raise("RuntimeError", "unterminated dumped string") + } + + enc := s.EncName() + if rest := data[i:]; len(rest) > 0 { + enc = vm.undumpForceEncoding(string(rest)) + } + return object.NewStringBytesEnc(out, enc) +} + +// hexAt returns the hex value of data[i], reporting false when i is out of range +// or the byte is not a hex digit. +func hexAt(data []byte, i int) (int, bool) { + if i < 0 || i >= len(data) { + return 0, false + } + return undumpHexNibble(data[i]) +} + +// undumpUnicode decodes a \u escape at *i (already past the 'u'): either exactly +// four hex digits (\uXXXX) or one or more space-separated hex code points in +// braces (\u{XXXX YYYY}). Each code point is appended to out as UTF-8. A +// malformed escape raises RuntimeError. +func (vm *VM) undumpUnicode(data []byte, i *int, out []byte) []byte { + bad := func() { raise("RuntimeError", "invalid Unicode escape") } + if *i < len(data) && data[*i] == '{' { + *i++ + for { + for *i < len(data) && data[*i] == ' ' { + *i++ + } + if *i < len(data) && data[*i] == '}' { + *i++ + return out + } + start := *i + cp := 0 + for *i < len(data) { + v, ok := undumpHexNibble(data[*i]) + if !ok { + break + } + cp = cp<<4 | v + if cp > 0x10FFFF { + cp = 0x110000 // clamp so many digits cannot overflow int + } + *i++ + } + if *i == start { + bad() + } + if cp > 0x10FFFF { + raise("RuntimeError", "invalid Unicode codepoint (too large)") + } + out = utf8.AppendRune(out, rune(cp)) + } + } + cp := 0 + for k := 0; k < 4; k++ { + v, ok := hexAt(data, *i) + if !ok { + bad() + } + cp = cp<<4 | v + *i++ + } + return utf8.AppendRune(out, rune(cp)) +} + +// undumpForceEncoding parses the .force_encoding("NAME") suffix that follows the +// closing quote of a dumped non-default-encoding string and returns NAME. A +// malformed suffix, or a name no known encoding matches, raises RuntimeError. +func (vm *VM) undumpForceEncoding(rest string) string { + const pre = `.force_encoding("` + if !strings.HasPrefix(rest, pre) || !strings.HasSuffix(rest, `")`) || len(rest) < len(pre)+2 { + raise("RuntimeError", `invalid dumped string; not wrapped with '"' nor '"...".force_encoding("...")' form`) + } + name := rest[len(pre) : len(rest)-2] + if _, ok := vm.findEncoding(name); !ok { + raise("RuntimeError", "dumped string has unknown encoding name") + } + return name +} diff --git a/internal/vm/string_undump_internal_test.go b/internal/vm/string_undump_internal_test.go new file mode 100644 index 00000000..4471dfba --- /dev/null +++ b/internal/vm/string_undump_internal_test.go @@ -0,0 +1,88 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import "testing" + +// TestStringUndump covers String#undump parsing a String#dump literal back into +// the original string: the named escapes, \" \\ \#, \xHH, \uXXXX and \u{…} +// (single, multiple, and empty), printable passthrough, the .force_encoding +// suffix, and a String#dump round trip — including a non-ASCII-compatible source. +// Verified against ruby 4.0.6. +func TestStringUndump(t *testing.T) { + cases := []struct{ src, want string }{ + {`p '"foo"'.undump`, `"foo"`}, + {`p '""'.undump`, `""`}, + // Named escapes decode to their control byte. + {`p '"\a\b\t\n\v\f\r\e"'.undump.bytes`, `[7, 8, 9, 10, 11, 12, 13, 27]`}, + // \" \\ and \# unescape. + {`p "\"\\\"\\\\\"".undump`, `"\"\\"`}, + {`p '"\#{a}"'.undump`, `"\#{a}"`}, + // # is literal when not part of an escape. + {`p '"#1 # #a"'.undump`, `"#1 # #a"`}, + // \xHH (either case) becomes the byte. + {`p '"\x41\x7F"'.undump`, `"A\u007F"`}, + {`p '"\xab"'.undump.bytes`, `[171]`}, + // \uXXXX and \u{…} become UTF-8 code points. + {`p '"A"'.undump`, `"A"`}, + {`p '"\u00E9"'.undump`, `"é"`}, + {`p '"\u{80}"'.undump.bytes`, `[194, 128]`}, + {`p '"\u{41 42 43}"'.undump`, `"ABC"`}, + {`p '"\u{}"'.undump`, `""`}, + // An unrecognized escape keeps its backslash. + {`p '"\q"'.undump`, `"\\q"`}, + // A String#dump round trip is the identity. + {`p "café\t\"x\" ok".dump.undump == "café\t\"x\" ok"`, `true`}, + // Round trip through a non-ASCII-compatible encoding. + {`s = "\u{876}".encode('utf-16be'); p s.dump.undump == s`, `true`}, + {`p '"\bv".force_encoding("UTF-16BE")'.undump == "ࡶ".encode('utf-16be')`, `true`}, + // The result keeps the receiver's (ASCII-compatible) encoding. + {`p '"foo"'.encode("ISO-8859-1").undump.encoding.name`, `"ISO-8859-1"`}, + // dump ignores frozen and undump returns a plain String instance. + {`p '"foo"'.freeze.undump.frozen?`, `false`}, + {`p '"foo"'.undump.instance_of?(String)`, `true`}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want+"\n" { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n") + } + } +} + +// TestStringUndumpErrors covers every RuntimeError (and the CompatibilityError) +// String#undump raises: a missing or excess wrapping quote, a malformed \x or \u +// escape, an out-of-range code point, a raw NUL or non-ASCII byte, a trailing +// backslash, and a malformed or unknown .force_encoding suffix. Verified against +// ruby 4.0.6. +func TestStringUndumpErrors(t *testing.T) { + cases := []struct{ src, want string }{ + {`begin; 'foo'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`}, + {`begin; '"foo'.undump; rescue => e; p e.message; end`, `"unterminated dumped string"`}, + {`begin; 'foo"'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`}, + {`begin; '" "" "'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`}, + {`begin; '"\x"'.undump; rescue => e; p e.message; end`, `"invalid hex escape"`}, + {`begin; '"\x3y"'.undump; rescue => e; p e.message; end`, `"invalid hex escape"`}, + {`begin; '"\u"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`}, + {`begin; '"\u{"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`}, + {`begin; '"\u{3042"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`}, + {`begin; '"\u{zz}"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`}, + {`begin; '"\u{110000}"'.undump; rescue => e; p e.message; end`, `"invalid Unicode codepoint (too large)"`}, + {`begin; '"\z'.undump; rescue => e; p e.message; end`, `"unterminated dumped string"`}, + {`begin; s = %Q{"foo\0"}; s.undump; rescue => e; p e.message; end`, `"string contains null byte"`}, + {`begin; s = %Q{"あ"}; s.undump; rescue => e; p e.message; end`, `"non-ASCII character detected"`}, + {`begin; '"".force_encoding("Unknown")'.undump; rescue => e; p e.message; end`, `"dumped string has unknown encoding name"`}, + {`begin; '"".force_encoding("BINARY"'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`}, + {`begin; '"".force_encoding()'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`}, + // A trailing backslash with no following byte is an invalid escape. + {`begin; s = %Q{"\\}; s.undump; rescue => e; p e.message; end`, `"invalid escape"`}, + // A non-ASCII-compatible receiver cannot be undumped. + {`begin; '"foo"'.encode("utf-16le").undump; rescue => e; p e.class; end`, `Encoding::CompatibilityError`}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want+"\n" { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n") + } + } +}