Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 111 additions & 19 deletions internal/vm/io.go
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ func (vm *VM) registerIO() {
cFile := vm.consts["File"].(*RClass)
cFile.super = cIO // File < IO, inheriting the read+write protocol; is_a?(IO) holds
cFile.smethods["open"] = &Method{name: "open", owner: cFile, native: func(vm *VM, _ object.Value, args []object.Value, blk *Proc) object.Value {
o := openFileIO(cFile, pathArg(vm, args[0]), fileMode(args))
o := vm.openFileArgs(cFile, args) // openFileArgs rejects a missing path
if blk != nil {
defer ioFlushClose(o)
return vm.callBlock(blk, []object.Value{o})
Expand All @@ -423,7 +423,7 @@ func (vm *VM) registerIO() {
if len(args) == 0 {
raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)")
}
return openFileIO(cFile, pathArg(vm, args[0]), fileMode(args))
return vm.openFileArgs(cFile, args)
}}
// File instance metadata operations. Puppet's replace_file writes to a
// Uniquefile (a DelegateClass(File)) and then chmod/chowns it before renaming
Expand Down Expand Up @@ -524,19 +524,6 @@ func flagsToMode(flags int64) string {
}
}

// fileMode returns the access mode argument of File.open (default "r"). The mode
// may be a string ("w", "r+", ...) or an integer bit-OR of File::Constants flags
// (e.g. File::RDWR | File::CREAT | File::EXCL); a trailing opts Hash is ignored.
func fileMode(args []object.Value) string {
if len(args) > 1 {
if i, ok := args[1].(object.Integer); ok {
return flagsToMode(int64(i))
}
return strArg(args[1])
}
return "r"
}

// stringIOModeVal resolves a StringIO mode argument to either an Integer flag set
// or a String access mode: Integer and String are taken directly, and any other
// object is converted via #to_str (raising TypeError otherwise). Keeping the
Expand Down Expand Up @@ -596,6 +583,56 @@ func stringIOModeFlags(mode string) (read, write, trunc, appnd bool) {
return false, false, false, false
}

// openFileArgs opens a file for File.open / File.new / Kernel#open, following
// io.c rb_scan_args "12:": the arguments are (path, [mode], [perm], **opts) with
// a trailing Hash taken as options rather than the mode. The access mode comes
// from the positional mode (a String or an Integer flag set) or the :mode
// option; the external/internal encoding comes from the mode string's
// ":ext[:int]" suffix or the :encoding / :external_encoding / :internal_encoding
// options (rb_io_extract_modeenc), and is recorded on the stream so reads honour
// it. A trailing Hash mode argument is thus no longer mistaken for a mode String.
func (vm *VM) openFileArgs(cls *RClass, args []object.Value) *IOObj {
pos, opts := splitIOOpts(args)
if len(pos) == 0 {
raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)")
}
mode := "r"
if len(pos) > 1 && !object.IsNil(pos[1]) {
mode = vm.vmodeString(pos[1])
} else if opts != nil {
if m, ok := opts.Get(object.Symbol("mode")); ok && !object.IsNil(m) {
mode = vm.vmodeString(m)
}
}
o := openFileIO(cls, pathArg(vm, pos[0]), mode)
ms := vm.ioResolveModeEnc(pos, opts)
o.extEnc, o.intEnc, o.binmode = ms.extEnc, ms.intEnc, ms.binmode
return o
}

// vmodeString reduces a File.open mode argument to the fopen-style base mode
// string openFileIO understands: an Integer (or #to_int object) is mapped through
// flagsToMode; a String is taken as-is (its ":enc" suffix is tolerated and later
// resolved for encoding); any other object is coerced with #to_str.
func (vm *VM) vmodeString(v object.Value) string {
if i, ok := v.(object.Integer); ok {
return flagsToMode(int64(i))
}
if s, ok := v.(*object.String); ok {
return s.Str()
}
if vm.respondsToDynamic(v, "to_int") {
return flagsToMode(vm.repeatLong(v))
}
if vm.respondsToDynamic(v, "to_str") {
if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok {
return s.Str()
}
}
raise("TypeError", "no implicit conversion of %s into String", classNameOf(v))
return ""
}

// openFileIO opens path into a buffered, file-backed IOObj per mode (r/w/a, with
// an optional "+" making a read mode writable). The file's bytes are read into
// the buffer; writes accumulate there and are flushed back on flush/close.
Expand Down Expand Up @@ -897,6 +934,46 @@ func defIOWrite(cls *RClass) {
})
}

// ioReadEnc returns the (external, internal) encoding names in effect for a
// whole-stream read, filling the unset sides from Encoding.default_external and
// Encoding.default_internal. A BINARY external encoding — or an internal equal
// to the external — suppresses transcoding (io.c rb_io_ext_int_to_enc leaves the
// second converter NULL), so the returned internal name is "".
func (vm *VM) ioReadEnc(o *IOObj) (ext, intn string) {
ext = o.extEnc
if ext == "" && vm.defExternalEnc != nil {
ext = vm.defExternalEnc.name
}
intn = o.intEnc
if intn == "" && vm.defInternalEnc != nil {
intn = vm.defInternalEnc.name
}
if ext == "ASCII-8BIT" || intn == ext {
intn = ""
}
return ext, intn
}

// ioDecodeRead builds the String a whole-stream read (or a gets-family line)
// returns from raw external bytes: when an internal encoding is in effect the
// bytes are transcoded external→internal and tagged with it; otherwise they are
// tagged with the external encoding unchanged (io.c io_enc_str).
func (vm *VM) ioDecodeRead(o *IOObj, data []byte) *object.String {
ext, intn := vm.ioReadEnc(o)
b := append([]byte(nil), data...)
if intn == "" {
return object.NewStringBytesEnc(b, ext)
}
// Pure-ASCII content is byte-identical across ASCII-compatible encodings, so it
// is simply retagged with the internal encoding — no converter needed (this is
// how MRI transcodes e.g. IBM866→UTF-8 for an ASCII line).
if asciiOnly(b) && vm.encAsciiCompat(ext) && vm.encAsciiCompat(intn) {
return object.NewStringBytesEnc(b, intn)
}
src := object.NewStringBytesEnc(b, ext)
return vm.stringEncode(src, []object.Value{object.NewString(intn)})
}

// defStringIORead defines the reading half of the protocol, plus the cursor and
// content methods, on StringIO.
func defStringIORead(cls *RClass) {
Expand Down Expand Up @@ -1023,21 +1100,30 @@ func defStringIORead(cls *RClass) {
raise("FrozenError", "can't modify frozen String: %s", buf.Inspect())
}
if isNil {
buf.SetBytes(nil)
buf.SetBytes(nil) // read at EOF clears the buffer, leaving its encoding
return object.NilV
}
buf.SetBytes(append([]byte(nil), data...))
if lengthGiven {
// read(size, buf): the bytes are binary and the buffer's own
// encoding is left unchanged (io.c io_read: no transcoding path).
buf.SetBytes(append([]byte(nil), data...))
return buf
}
dec := vm.ioDecodeRead(o, data)
buf.SetBytes(append([]byte(nil), dec.Bytes()...))
buf.Enc = dec.Enc // a full read retags the buffer with the read encoding
return buf
}
if isNil {
return object.NilV
}
// read(length) returns a binary (ASCII-8BIT) String; read with no
// length returns the remainder in the stream's default encoding.
// length returns the remainder in the stream's read encoding, transcoded
// external→internal when an internal encoding is in effect.
if lengthGiven {
return object.NewStringBytesEnc(append([]byte(nil), data...), "ASCII-8BIT")
}
return object.NewStringBytes(append([]byte(nil), data...))
return vm.ioDecodeRead(o, data)
}
if lengthGiven {
n := int(vm.toIntCoerce(args[0]))
Expand Down Expand Up @@ -1506,6 +1592,12 @@ func (vm *VM) ioGetsResolved(o *IOObj, sep getsSep, limit int, chomp bool) objec
v := ioGetsLine(o, sep, limit, chomp)
if v != object.NilV {
o.lineno++
// A line is read in the external encoding (the separator is matched on those
// bytes) then transcoded to the internal encoding and tagged, exactly like a
// whole-stream read (io.c rb_io_getline_1 → io_enc_str).
if s, ok := v.(*object.String); ok {
v = vm.ioDecodeRead(o, s.Bytes())
}
}
return v
}
Expand Down
20 changes: 19 additions & 1 deletion internal/vm/io_class.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (vm *VM) registerIOClassMethods(cIO, cFile *RClass) {
path := pathArg(vm, args[0])
mode := "r"
if len(args) >= 2 && !object.IsNil(args[1]) {
mode = fileMode(args[:2])
mode = vm.vmodeString(args[1]) // Integer flag set or fopen-style String
}
o := openFileIO(cFile, path, mode)
vm.nextFd++
Expand Down Expand Up @@ -305,6 +305,14 @@ func (vm *VM) ioReadFile(args []object.Value, forceBinary bool) object.Value {
start = int64(len(data))
}
out := data[start:]
// A "BOM|enc" external encoding (mode "rb:BOM|utf-8") strips a leading BOM from
// the file's start when one is present (io.c: the BOM overrides the named
// encoding). Only a read from offset 0 can see the BOM.
if start == 0 && modeHasBOM(mode) {
if n, bomEnc := detectBOM(out); bomEnc != "" {
out = out[n:]
}
}
if hasLen && int64(len(out)) > length {
out = out[:length]
}
Expand Down Expand Up @@ -466,6 +474,16 @@ func ioModeWritable(mode string) bool {
return !strings.HasPrefix(base, "r") || strings.Contains(base, "+")
}

// modeHasBOM reports whether a mode string's encoding part carries the "BOM|"
// prefix (e.g. "rb:BOM|utf-8"), which asks IO.read/File.read to strip and honour
// a leading byte-order mark.
func modeHasBOM(mode string) bool {
if i := strings.IndexByte(mode, ':'); i >= 0 {
return strings.HasPrefix(strings.ToUpper(mode[i+1:]), "BOM|")
}
return false
}

// modeBase strips a ":enc" encoding suffix (e.g. "w:UTF-16LE") from a mode string.
func modeBase(mode string) string {
if i := strings.IndexByte(mode, ':'); i >= 0 {
Expand Down
88 changes: 84 additions & 4 deletions internal/vm/io_descriptors.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package vm

import (
"math/big"
"unicode/utf8"
"os"

"github.com/go-embedded-ruby/ruby/internal/object"
)
Expand Down Expand Up @@ -39,16 +39,26 @@ func defIOReadExtra(cls *RClass) {
o.pos++
return object.IntValue(int64(b))
})
cls.define("readchar", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
cls.define("readchar", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
o := self.(*IOObj)
ioCheckReadable(o)
o.pipeRefresh()
if o.pos >= len(o.buf) {
raise("EOFError", "end of file reached")
}
r, sz := utf8.DecodeRune(o.buf[o.pos:])
// One character is a full character of the stream's external encoding — a
// multi-byte EUC-JP/UTF-8 char, not a single byte — which is then transcoded
// to the internal encoding when one is set (io.c io_getc → read_all path).
// decodeCharFrom never reports more bytes than remain (an incomplete lead
// yields 0, taken as a one-byte character), so the slice below is in bounds.
ext, _ := vm.ioReadEnc(o)
_, sz, _, _ := vm.decodeCharFrom(o.buf[o.pos:], ext)
if sz < 1 {
sz = 1 // an invalid or truncated lead byte is a one-byte character
}
charBytes := o.buf[o.pos : o.pos+sz]
o.pos += sz
return object.NewString(string(r))
return vm.ioDecodeRead(o, charBytes)
})
cls.define("ungetbyte", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value {
o := self.(*IOObj)
Expand Down Expand Up @@ -292,6 +302,76 @@ func defIOSeekable(cls *RClass) {
cls.define("fdatasync", func(_ *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value {
return object.IntValue(0)
})
// set_encoding_by_bom (io.c rb_io_set_encoding_by_bom): if the stream begins
// with a Unicode byte-order mark, consume it and set the external encoding to
// the one the BOM names, returning that Encoding; otherwise leave the stream
// untouched and return nil. The stream must be in binary mode with no encoding
// already set (ArgumentError otherwise), and a non-readable stream simply
// returns nil.
cls.define("set_encoding_by_bom", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
o := self.(*IOObj)
if o.closed { // GetOpenFile: a closed stream raises IOError first
raise("IOError", "closed stream")
}
if !o.binmode {
raise("ArgumentError", "ASCII incompatible encoding needs binmode")
}
if o.intEnc != "" {
raise("ArgumentError", "encoding conversion is set")
}
if o.extEnc != "" && o.extEnc != "ASCII-8BIT" {
raise("ArgumentError", "encoding is set to %s already", o.extEnc)
}
if o.rdClosed { // a write-only stream is not readable: no BOM to strip
return object.NilV
}
// rbgo buffers a file's bytes at open; refresh a read-only file-backed
// stream from disk so a BOM written after the open (as the specs do) is
// visible, mirroring MRI's lazy read from the descriptor.
if o.path != "" && !o.writable {
if b, err := os.ReadFile(o.path); err == nil {
o.buf = b
}
}
enc := ioStripBOM(o)
if enc == "" {
return object.NilV
}
o.extEnc = enc
e, _ := vm.findEncoding(enc) // every BOM name is a registered encoding
return e
})
}

// ioStripBOM inspects the bytes at the cursor for a Unicode byte-order mark and,
// on a full match, advances the cursor past it and returns the encoding name the
// BOM designates ("" when none is found). It mirrors io.c io_strip_bom: a
// truncated BOM leaves the cursor where it was, and "\xFF\xFE" followed by two
// NUL bytes is UTF-32LE while "\xFF\xFE" alone is UTF-16LE.
func ioStripBOM(o *IOObj) string {
n, enc := detectBOM(o.buf[o.pos:])
o.pos += n
return enc
}

// detectBOM reports the byte length and encoding name of a leading Unicode
// byte-order mark in p (0, "" when there is none). "\xFF\xFE" followed by two
// NUL bytes is UTF-32LE, while "\xFF\xFE" alone is UTF-16LE (io.c io_strip_bom).
func detectBOM(p []byte) (int, string) {
switch {
case len(p) >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF:
return 3, "UTF-8"
case len(p) >= 2 && p[0] == 0xFF && p[1] == 0xFE:
if len(p) >= 4 && p[2] == 0x00 && p[3] == 0x00 {
return 4, "UTF-32LE"
}
return 2, "UTF-16LE"
case len(p) >= 2 && p[0] == 0xFE && p[1] == 0xFF:
return 2, "UTF-16BE"
case len(p) >= 4 && p[0] == 0x00 && p[1] == 0x00 && p[2] == 0xFE && p[3] == 0xFF:
return 4, "UTF-32BE"
}
return 0, ""
}

// ioUnget inserts p immediately before the cursor (leaving the cursor on the
Expand Down
Loading