Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions internal/vm/builtins.go
Original file line number Diff line number Diff line change
Expand Up @@ -1465,12 +1465,21 @@ func (vm *VM) bootstrap() {
vm.cString.define("empty?", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
return object.Bool(len(strOf(self)) == 0)
})
vm.cString.define("dump", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
// Always a plain (never frozen, never subclass) String in the receiver's
// encoding; a non-ASCII-compatible source additionally carries a
// .force_encoding suffix inside the dumped text (see (*String).Dump).
vm.cString.define("dump", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
// Always a plain (never frozen, never subclass) String. The escaped text is
// pure ASCII, so it keeps the receiver's encoding when that is ASCII-
// compatible, but a non-ASCII-compatible source degrades to US-ASCII and
// carries a .force_encoding suffix inside the dumped text (see
// (*String).Dump) — this is what lets String#undump read it back.
s := self.(*object.String)
return object.NewStringBytesEnc([]byte(s.Dump()), s.EncName())
enc := s.EncName()
if e, ok := vm.findEncoding(enc); ok && !e.asciiCompat {
enc = "US-ASCII"
}
return object.NewStringBytesEnc([]byte(s.Dump()), enc)
})
vm.cString.define("undump", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
return vm.stringUndump(self.(*object.String))
})
vm.cString.define("upcase", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
return strEncOf(self, strings.ToUpper(strOf(self)))
Expand Down
189 changes: 189 additions & 0 deletions internal/vm/string_undump.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
// Copyright (c) the go-embedded-ruby/ruby authors
//
// SPDX-License-Identifier: BSD-3-Clause

package vm

import (
"strings"
"unicode/utf8"

"github.com/go-embedded-ruby/ruby/internal/object"
)

// undumpNamedEscape maps the backslash-escaped control letters String#undump
// recognizes back to their byte value (the inverse of Dump's named escapes).
var undumpNamedEscape = map[byte]byte{
'a': 0x07, 'b': 0x08, 't': 0x09, 'n': 0x0a,
'v': 0x0b, 'f': 0x0c, 'r': 0x0d, 'e': 0x1b,
'"': '"', '\\': '\\', '#': '#',
}

// undumpHexNibble returns the value of a single hexadecimal digit and whether c is one.
func undumpHexNibble(c byte) (int, bool) {
switch {
case c >= '0' && c <= '9':
return int(c - '0'), true
case c >= 'a' && c <= 'f':
return int(c-'a') + 10, true
case c >= 'A' && c <= 'F':
return int(c-'A') + 10, true
}
return 0, false
}

// stringUndump implements String#undump: it parses a String#dump literal back
// into the original string. The receiver must be an ASCII-compatible encoding
// (else Encoding::CompatibilityError); the text must be wrapped in double quotes,
// optionally followed by .force_encoding("NAME"); every backslash escape is
// decoded; and a raw NUL or non-ASCII byte, a malformed escape, or trailing junk
// raises RuntimeError. Verified against ruby 4.0.6.
func (vm *VM) stringUndump(s *object.String) object.Value {
if e, ok := vm.findEncoding(s.EncName()); ok && !e.asciiCompat {
raise("Encoding::CompatibilityError", "ASCII incompatible encoding: %s", s.EncName())
}
data := s.Bytes()
n := len(data)
invalid := func() {
raise("RuntimeError", `invalid dumped string; not wrapped with '"' nor '"...".force_encoding("...")' form`)
}
if n == 0 || data[0] != '"' {
invalid()
}

var out []byte
i := 1
closed := false
for i < n {
c := data[i]
if c == '"' {
i++
closed = true
break
}
if c == 0 {
raise("RuntimeError", "string contains null byte")
}
if c >= 0x80 {
raise("RuntimeError", "non-ASCII character detected")
}
if c != '\\' {
out = append(out, c)
i++
continue
}
// A backslash escape.
i++
if i >= n {
raise("RuntimeError", "invalid escape")
}
esc := data[i]
if b, ok := undumpNamedEscape[esc]; ok {
out = append(out, b)
i++
continue
}
switch esc {
case 'x':
i++
hi, ok1 := hexAt(data, i)
lo, ok2 := hexAt(data, i+1)
if !ok1 || !ok2 {
raise("RuntimeError", "invalid hex escape")
}
out = append(out, byte(hi<<4|lo))
i += 2
case 'u':
i++
out = vm.undumpUnicode(data, &i, out)
default:
// dump never emits any other escape; MRI leaves an unrecognized one
// as-is, keeping the backslash (\q undumps to \q).
out = append(out, '\\', esc)
i++
}
}
if !closed {
raise("RuntimeError", "unterminated dumped string")
}

enc := s.EncName()
if rest := data[i:]; len(rest) > 0 {
enc = vm.undumpForceEncoding(string(rest))
}
return object.NewStringBytesEnc(out, enc)
}

// hexAt returns the hex value of data[i], reporting false when i is out of range
// or the byte is not a hex digit.
func hexAt(data []byte, i int) (int, bool) {
if i < 0 || i >= len(data) {
return 0, false
}
return undumpHexNibble(data[i])
}

// undumpUnicode decodes a \u escape at *i (already past the 'u'): either exactly
// four hex digits (\uXXXX) or one or more space-separated hex code points in
// braces (\u{XXXX YYYY}). Each code point is appended to out as UTF-8. A
// malformed escape raises RuntimeError.
func (vm *VM) undumpUnicode(data []byte, i *int, out []byte) []byte {
bad := func() { raise("RuntimeError", "invalid Unicode escape") }
if *i < len(data) && data[*i] == '{' {
*i++
for {
for *i < len(data) && data[*i] == ' ' {
*i++
}
if *i < len(data) && data[*i] == '}' {
*i++
return out
}
start := *i
cp := 0
for *i < len(data) {
v, ok := undumpHexNibble(data[*i])
if !ok {
break
}
cp = cp<<4 | v
if cp > 0x10FFFF {
cp = 0x110000 // clamp so many digits cannot overflow int
}
*i++
}
if *i == start {
bad()
}
if cp > 0x10FFFF {
raise("RuntimeError", "invalid Unicode codepoint (too large)")
}
out = utf8.AppendRune(out, rune(cp))
}
}
cp := 0
for k := 0; k < 4; k++ {
v, ok := hexAt(data, *i)
if !ok {
bad()
}
cp = cp<<4 | v
*i++
}
return utf8.AppendRune(out, rune(cp))
}

// undumpForceEncoding parses the .force_encoding("NAME") suffix that follows the
// closing quote of a dumped non-default-encoding string and returns NAME. A
// malformed suffix, or a name no known encoding matches, raises RuntimeError.
func (vm *VM) undumpForceEncoding(rest string) string {
const pre = `.force_encoding("`
if !strings.HasPrefix(rest, pre) || !strings.HasSuffix(rest, `")`) || len(rest) < len(pre)+2 {
raise("RuntimeError", `invalid dumped string; not wrapped with '"' nor '"...".force_encoding("...")' form`)
}
name := rest[len(pre) : len(rest)-2]
if _, ok := vm.findEncoding(name); !ok {
raise("RuntimeError", "dumped string has unknown encoding name")
}
return name
}
88 changes: 88 additions & 0 deletions internal/vm/string_undump_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright (c) the go-embedded-ruby/ruby authors
//
// SPDX-License-Identifier: BSD-3-Clause

package vm

import "testing"

// TestStringUndump covers String#undump parsing a String#dump literal back into
// the original string: the named escapes, \" \\ \#, \xHH, \uXXXX and \u{…}
// (single, multiple, and empty), printable passthrough, the .force_encoding
// suffix, and a String#dump round trip — including a non-ASCII-compatible source.
// Verified against ruby 4.0.6.
func TestStringUndump(t *testing.T) {
cases := []struct{ src, want string }{
{`p '"foo"'.undump`, `"foo"`},
{`p '""'.undump`, `""`},
// Named escapes decode to their control byte.
{`p '"\a\b\t\n\v\f\r\e"'.undump.bytes`, `[7, 8, 9, 10, 11, 12, 13, 27]`},
// \" \\ and \# unescape.
{`p "\"\\\"\\\\\"".undump`, `"\"\\"`},
{`p '"\#{a}"'.undump`, `"\#{a}"`},
// # is literal when not part of an escape.
{`p '"#1 # #a"'.undump`, `"#1 # #a"`},
// \xHH (either case) becomes the byte.
{`p '"\x41\x7F"'.undump`, `"A\u007F"`},
{`p '"\xab"'.undump.bytes`, `[171]`},
// \uXXXX and \u{…} become UTF-8 code points.
{`p '"A"'.undump`, `"A"`},
{`p '"\u00E9"'.undump`, `"é"`},
{`p '"\u{80}"'.undump.bytes`, `[194, 128]`},
{`p '"\u{41 42 43}"'.undump`, `"ABC"`},
{`p '"\u{}"'.undump`, `""`},
// An unrecognized escape keeps its backslash.
{`p '"\q"'.undump`, `"\\q"`},
// A String#dump round trip is the identity.
{`p "café\t\"x\" ok".dump.undump == "café\t\"x\" ok"`, `true`},
// Round trip through a non-ASCII-compatible encoding.
{`s = "\u{876}".encode('utf-16be'); p s.dump.undump == s`, `true`},
{`p '"\bv".force_encoding("UTF-16BE")'.undump == "ࡶ".encode('utf-16be')`, `true`},
// The result keeps the receiver's (ASCII-compatible) encoding.
{`p '"foo"'.encode("ISO-8859-1").undump.encoding.name`, `"ISO-8859-1"`},
// dump ignores frozen and undump returns a plain String instance.
{`p '"foo"'.freeze.undump.frozen?`, `false`},
{`p '"foo"'.undump.instance_of?(String)`, `true`},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want+"\n" {
t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n")
}
}
}

// TestStringUndumpErrors covers every RuntimeError (and the CompatibilityError)
// String#undump raises: a missing or excess wrapping quote, a malformed \x or \u
// escape, an out-of-range code point, a raw NUL or non-ASCII byte, a trailing
// backslash, and a malformed or unknown .force_encoding suffix. Verified against
// ruby 4.0.6.
func TestStringUndumpErrors(t *testing.T) {
cases := []struct{ src, want string }{
{`begin; 'foo'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`},
{`begin; '"foo'.undump; rescue => e; p e.message; end`, `"unterminated dumped string"`},
{`begin; 'foo"'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`},
{`begin; '" "" "'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`},
{`begin; '"\x"'.undump; rescue => e; p e.message; end`, `"invalid hex escape"`},
{`begin; '"\x3y"'.undump; rescue => e; p e.message; end`, `"invalid hex escape"`},
{`begin; '"\u"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`},
{`begin; '"\u{"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`},
{`begin; '"\u{3042"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`},
{`begin; '"\u{zz}"'.undump; rescue => e; p e.message; end`, `"invalid Unicode escape"`},
{`begin; '"\u{110000}"'.undump; rescue => e; p e.message; end`, `"invalid Unicode codepoint (too large)"`},
{`begin; '"\z'.undump; rescue => e; p e.message; end`, `"unterminated dumped string"`},
{`begin; s = %Q{"foo\0"}; s.undump; rescue => e; p e.message; end`, `"string contains null byte"`},
{`begin; s = %Q{"あ"}; s.undump; rescue => e; p e.message; end`, `"non-ASCII character detected"`},
{`begin; '"".force_encoding("Unknown")'.undump; rescue => e; p e.message; end`, `"dumped string has unknown encoding name"`},
{`begin; '"".force_encoding("BINARY"'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`},
{`begin; '"".force_encoding()'.undump; rescue => e; p e.message; end`, `"invalid dumped string; not wrapped with '\"' nor '\"...\".force_encoding(\"...\")' form"`},
// A trailing backslash with no following byte is an invalid escape.
{`begin; s = %Q{"\\}; s.undump; rescue => e; p e.message; end`, `"invalid escape"`},
// A non-ASCII-compatible receiver cannot be undumped.
{`begin; '"foo"'.encode("utf-16le").undump; rescue => e; p e.class; end`, `Encoding::CompatibilityError`},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want+"\n" {
t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n")
}
}
}