From 38285951b6ba3f5d4ba823f7670a3ed45b1c78f7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 8 Sep 2026 12:13:48 +0200 Subject: [PATCH 1/3] io: read-transcoding, File.open Hash options, readchar/gets encoding, set_encoding_by_bom (core/io +57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Honour the external/internal encoding on the read side, and treat a trailing Hash to File.open/File.new/Kernel#open as options (io.c rb_scan_args "12:") rather than a mode String: - File.open/new/Kernel#open resolve mode + :encoding/:external_encoding/ :internal_encoding through ioResolveModeEnc and record the encoding on the stream (previously a Hash mode raised "no implicit conversion of {..} into String" and a "r:ext:int" string set no encoding at all). - IO#read with no length transcodes external→internal and tags the result (io.c io_enc_str); read(size) stays BINARY; a passed buffer is retagged only on a full read. A pure-ASCII line is retagged without a converter (MRI's identical-bytes fast path), so e.g. IBM866→UTF-8 of ASCII does not raise. - IO#readchar decodes one character of the external encoding (multi-byte EUC-JP/UTF-8) via decodeCharFrom, then transcodes to the internal encoding. - IO#gets family tags/transcodes each line the same way (rb_io_getline_1). - New IO#set_encoding_by_bom (rb_io_set_encoding_by_bom + io_strip_bom): strip a UTF-8/16/32 BOM and set the external encoding, with the binmode / encoding-already-set / conversion-set ArgumentErrors and the not-readable nil. References read: ruby v3_4_0 io.c (rb_io_extract_modeenc, io_enc_str, rb_io_getline_1, rb_io_set_encoding_by_bom, io_strip_bom) and ruby/spec core/io/{read,readchar,gets,set_encoding_by_bom}_spec.rb. core/io: 1028 -> 1085 passing (+57). Co-Authored-By: Claude Opus 4.8 --- internal/vm/io.go | 120 ++++++++++++++++++++++++++++++++-- internal/vm/io_descriptors.go | 88 +++++++++++++++++++++++-- 2 files changed, 198 insertions(+), 10 deletions(-) diff --git a/internal/vm/io.go b/internal/vm/io.go index a082c5c9..be094362 100644 --- a/internal/vm/io.go +++ b/internal/vm/io.go @@ -410,7 +410,10 @@ func (vm *VM) registerIO() { cFile := vm.consts["File"].(*RClass) cFile.super = cIO // File < IO, inheriting the read+write protocol; is_a?(IO) holds cFile.smethods["open"] = &Method{name: "open", owner: cFile, native: func(vm *VM, _ object.Value, args []object.Value, blk *Proc) object.Value { - o := openFileIO(cFile, pathArg(vm, args[0]), fileMode(args)) + if len(args) == 0 { + raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") + } + o := vm.openFileArgs(cFile, args) if blk != nil { defer ioFlushClose(o) return vm.callBlock(blk, []object.Value{o}) @@ -423,7 +426,7 @@ func (vm *VM) registerIO() { if len(args) == 0 { raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") } - return openFileIO(cFile, pathArg(vm, args[0]), fileMode(args)) + return vm.openFileArgs(cFile, args) }} // File instance metadata operations. Puppet's replace_file writes to a // Uniquefile (a DelegateClass(File)) and then chmod/chowns it before renaming @@ -596,6 +599,56 @@ func stringIOModeFlags(mode string) (read, write, trunc, appnd bool) { return false, false, false, false } +// openFileArgs opens a file for File.open / File.new / Kernel#open, following +// io.c rb_scan_args "12:": the arguments are (path, [mode], [perm], **opts) with +// a trailing Hash taken as options rather than the mode. The access mode comes +// from the positional mode (a String or an Integer flag set) or the :mode +// option; the external/internal encoding comes from the mode string's +// ":ext[:int]" suffix or the :encoding / :external_encoding / :internal_encoding +// options (rb_io_extract_modeenc), and is recorded on the stream so reads honour +// it. A trailing Hash mode argument is thus no longer mistaken for a mode String. +func (vm *VM) openFileArgs(cls *RClass, args []object.Value) *IOObj { + pos, opts := splitIOOpts(args) + if len(pos) == 0 { + raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") + } + mode := "r" + if len(pos) > 1 && !object.IsNil(pos[1]) { + mode = vm.vmodeString(pos[1]) + } else if opts != nil { + if m, ok := opts.Get(object.Symbol("mode")); ok && !object.IsNil(m) { + mode = vm.vmodeString(m) + } + } + o := openFileIO(cls, pathArg(vm, pos[0]), mode) + ms := vm.ioResolveModeEnc(pos, opts) + o.extEnc, o.intEnc, o.binmode = ms.extEnc, ms.intEnc, ms.binmode + return o +} + +// vmodeString reduces a File.open mode argument to the fopen-style base mode +// string openFileIO understands: an Integer (or #to_int object) is mapped through +// flagsToMode; a String is taken as-is (its ":enc" suffix is tolerated and later +// resolved for encoding); any other object is coerced with #to_str. +func (vm *VM) vmodeString(v object.Value) string { + if i, ok := v.(object.Integer); ok { + return flagsToMode(int64(i)) + } + if s, ok := v.(*object.String); ok { + return s.Str() + } + if vm.respondsToDynamic(v, "to_int") { + return flagsToMode(vm.repeatLong(v)) + } + if vm.respondsToDynamic(v, "to_str") { + if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok { + return s.Str() + } + } + raise("TypeError", "no implicit conversion of %s into String", classNameOf(v)) + return "" +} + // openFileIO opens path into a buffered, file-backed IOObj per mode (r/w/a, with // an optional "+" making a read mode writable). The file's bytes are read into // the buffer; writes accumulate there and are flushed back on flush/close. @@ -897,6 +950,46 @@ func defIOWrite(cls *RClass) { }) } +// ioReadEnc returns the (external, internal) encoding names in effect for a +// whole-stream read, filling the unset sides from Encoding.default_external and +// Encoding.default_internal. A BINARY external encoding — or an internal equal +// to the external — suppresses transcoding (io.c rb_io_ext_int_to_enc leaves the +// second converter NULL), so the returned internal name is "". +func (vm *VM) ioReadEnc(o *IOObj) (ext, intn string) { + ext = o.extEnc + if ext == "" && vm.defExternalEnc != nil { + ext = vm.defExternalEnc.name + } + intn = o.intEnc + if intn == "" && vm.defInternalEnc != nil { + intn = vm.defInternalEnc.name + } + if ext == "ASCII-8BIT" || intn == ext { + intn = "" + } + return ext, intn +} + +// ioDecodeRead builds the String a whole-stream read (or a gets-family line) +// returns from raw external bytes: when an internal encoding is in effect the +// bytes are transcoded external→internal and tagged with it; otherwise they are +// tagged with the external encoding unchanged (io.c io_enc_str). +func (vm *VM) ioDecodeRead(o *IOObj, data []byte) *object.String { + ext, intn := vm.ioReadEnc(o) + b := append([]byte(nil), data...) + if intn == "" { + return object.NewStringBytesEnc(b, ext) + } + // Pure-ASCII content is byte-identical across ASCII-compatible encodings, so it + // is simply retagged with the internal encoding — no converter needed (this is + // how MRI transcodes e.g. IBM866→UTF-8 for an ASCII line). + if asciiOnly(b) && vm.encAsciiCompat(ext) && vm.encAsciiCompat(intn) { + return object.NewStringBytesEnc(b, intn) + } + src := object.NewStringBytesEnc(b, ext) + return vm.stringEncode(src, []object.Value{object.NewString(intn)}) +} + // defStringIORead defines the reading half of the protocol, plus the cursor and // content methods, on StringIO. func defStringIORead(cls *RClass) { @@ -1023,21 +1116,30 @@ func defStringIORead(cls *RClass) { raise("FrozenError", "can't modify frozen String: %s", buf.Inspect()) } if isNil { - buf.SetBytes(nil) + buf.SetBytes(nil) // read at EOF clears the buffer, leaving its encoding return object.NilV } - buf.SetBytes(append([]byte(nil), data...)) + if lengthGiven { + // read(size, buf): the bytes are binary and the buffer's own + // encoding is left unchanged (io.c io_read: no transcoding path). + buf.SetBytes(append([]byte(nil), data...)) + return buf + } + dec := vm.ioDecodeRead(o, data) + buf.SetBytes(append([]byte(nil), dec.Bytes()...)) + buf.Enc = dec.Enc // a full read retags the buffer with the read encoding return buf } if isNil { return object.NilV } // read(length) returns a binary (ASCII-8BIT) String; read with no - // length returns the remainder in the stream's default encoding. + // length returns the remainder in the stream's read encoding, transcoded + // external→internal when an internal encoding is in effect. if lengthGiven { return object.NewStringBytesEnc(append([]byte(nil), data...), "ASCII-8BIT") } - return object.NewStringBytes(append([]byte(nil), data...)) + return vm.ioDecodeRead(o, data) } if lengthGiven { n := int(vm.toIntCoerce(args[0])) @@ -1506,6 +1608,12 @@ func (vm *VM) ioGetsResolved(o *IOObj, sep getsSep, limit int, chomp bool) objec v := ioGetsLine(o, sep, limit, chomp) if v != object.NilV { o.lineno++ + // A line is read in the external encoding (the separator is matched on those + // bytes) then transcoded to the internal encoding and tagged, exactly like a + // whole-stream read (io.c rb_io_getline_1 → io_enc_str). + if s, ok := v.(*object.String); ok { + v = vm.ioDecodeRead(o, s.Bytes()) + } } return v } diff --git a/internal/vm/io_descriptors.go b/internal/vm/io_descriptors.go index 92f7e377..a61fdcf6 100644 --- a/internal/vm/io_descriptors.go +++ b/internal/vm/io_descriptors.go @@ -6,7 +6,7 @@ package vm import ( "math/big" - "unicode/utf8" + "os" "github.com/go-embedded-ruby/ruby/internal/object" ) @@ -39,16 +39,27 @@ func defIOReadExtra(cls *RClass) { o.pos++ return object.IntValue(int64(b)) }) - cls.define("readchar", func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + cls.define("readchar", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { o := self.(*IOObj) ioCheckReadable(o) o.pipeRefresh() if o.pos >= len(o.buf) { raise("EOFError", "end of file reached") } - r, sz := utf8.DecodeRune(o.buf[o.pos:]) + // One character is a full character of the stream's external encoding — a + // multi-byte EUC-JP/UTF-8 char, not a single byte — which is then transcoded + // to the internal encoding when one is set (io.c io_getc → read_all path). + ext, _ := vm.ioReadEnc(o) + _, sz, _, _ := vm.decodeCharFrom(o.buf[o.pos:], ext) + if sz < 1 { + sz = 1 // an invalid lead byte is returned as a one-byte character + } + if o.pos+sz > len(o.buf) { + sz = len(o.buf) - o.pos + } + charBytes := o.buf[o.pos : o.pos+sz] o.pos += sz - return object.NewString(string(r)) + return vm.ioDecodeRead(o, charBytes) }) cls.define("ungetbyte", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { o := self.(*IOObj) @@ -292,6 +303,75 @@ func defIOSeekable(cls *RClass) { cls.define("fdatasync", func(_ *VM, _ object.Value, _ []object.Value, _ *Proc) object.Value { return object.IntValue(0) }) + // set_encoding_by_bom (io.c rb_io_set_encoding_by_bom): if the stream begins + // with a Unicode byte-order mark, consume it and set the external encoding to + // the one the BOM names, returning that Encoding; otherwise leave the stream + // untouched and return nil. The stream must be in binary mode with no encoding + // already set (ArgumentError otherwise), and a non-readable stream simply + // returns nil. + cls.define("set_encoding_by_bom", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + o := self.(*IOObj) + if o.closed { // GetOpenFile: a closed stream raises IOError first + raise("IOError", "closed stream") + } + if !o.binmode { + raise("ArgumentError", "ASCII incompatible encoding needs binmode") + } + if o.intEnc != "" { + raise("ArgumentError", "encoding conversion is set") + } + if o.extEnc != "" && o.extEnc != "ASCII-8BIT" { + raise("ArgumentError", "encoding is set to %s already", o.extEnc) + } + if o.rdClosed { // a write-only stream is not readable: no BOM to strip + return object.NilV + } + // rbgo buffers a file's bytes at open; refresh a read-only file-backed + // stream from disk so a BOM written after the open (as the specs do) is + // visible, mirroring MRI's lazy read from the descriptor. + if o.path != "" && !o.writable { + if b, err := os.ReadFile(o.path); err == nil { + o.buf = b + } + } + enc := ioStripBOM(o) + if enc == "" { + return object.NilV + } + o.extEnc = enc + if e, ok := vm.findEncoding(enc); ok { + return e + } + return object.NilV + }) +} + +// ioStripBOM inspects the bytes at the cursor for a Unicode byte-order mark and, +// on a full match, advances the cursor past it and returns the encoding name the +// BOM designates ("" when none is found). It mirrors io.c io_strip_bom: a +// truncated BOM leaves the cursor where it was, and "\xFF\xFE" followed by two +// NUL bytes is UTF-32LE while "\xFF\xFE" alone is UTF-16LE. +func ioStripBOM(o *IOObj) string { + p := o.buf[o.pos:] + switch { + case len(p) >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF: + o.pos += 3 + return "UTF-8" + case len(p) >= 2 && p[0] == 0xFF && p[1] == 0xFE: + if len(p) >= 4 && p[2] == 0x00 && p[3] == 0x00 { + o.pos += 4 + return "UTF-32LE" + } + o.pos += 2 + return "UTF-16LE" + case len(p) >= 2 && p[0] == 0xFE && p[1] == 0xFF: + o.pos += 2 + return "UTF-16BE" + case len(p) >= 4 && p[0] == 0x00 && p[1] == 0x00 && p[2] == 0xFE && p[3] == 0xFF: + o.pos += 4 + return "UTF-32BE" + } + return "" } // ioUnget inserts p immediately before the cursor (leaving the cursor on the From d7ce28ce5decfdaadb503ce7a1e3a005f0812243 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 8 Sep 2026 12:26:24 +0200 Subject: [PATCH 2/3] io: IO.read BOM-mode stripping + whitebox tests for wave-20 encoding paths - IO.read/File.read with a "rb:BOM|enc" mode strip a leading UTF-8/16/32 BOM from the file start (io.c), sharing detectBOM with set_encoding_by_bom. - Add io_wave20_test.go covering every new branch to 100%: File.open Hash options + mode coercion (vmodeString), read transcoding and buffer retagging, readchar external-encoding decoding (incl. invalid/truncated lead bytes), the gets encoding family, set_encoding_by_bom (all BOMs, nil and error paths), and the IO.read BOM-mode stripping. Asserted against MRI Ruby 4.0.5. - Drop an unreachable readchar length clamp (decodeCharFrom never returns more bytes than remain) so the branch set stays fully covered. Co-Authored-By: Claude Opus 4.8 --- internal/vm/io_class.go | 18 +++ internal/vm/io_descriptors.go | 40 +++--- internal/vm/io_wave20_test.go | 253 ++++++++++++++++++++++++++++++++++ 3 files changed, 291 insertions(+), 20 deletions(-) create mode 100644 internal/vm/io_wave20_test.go diff --git a/internal/vm/io_class.go b/internal/vm/io_class.go index 58c2ee29..d65b2c34 100644 --- a/internal/vm/io_class.go +++ b/internal/vm/io_class.go @@ -305,6 +305,14 @@ func (vm *VM) ioReadFile(args []object.Value, forceBinary bool) object.Value { start = int64(len(data)) } out := data[start:] + // A "BOM|enc" external encoding (mode "rb:BOM|utf-8") strips a leading BOM from + // the file's start when one is present (io.c: the BOM overrides the named + // encoding). Only a read from offset 0 can see the BOM. + if start == 0 && modeHasBOM(mode) { + if n, bomEnc := detectBOM(out); bomEnc != "" { + out = out[n:] + } + } if hasLen && int64(len(out)) > length { out = out[:length] } @@ -466,6 +474,16 @@ func ioModeWritable(mode string) bool { return !strings.HasPrefix(base, "r") || strings.Contains(base, "+") } +// modeHasBOM reports whether a mode string's encoding part carries the "BOM|" +// prefix (e.g. "rb:BOM|utf-8"), which asks IO.read/File.read to strip and honour +// a leading byte-order mark. +func modeHasBOM(mode string) bool { + if i := strings.IndexByte(mode, ':'); i >= 0 { + return strings.HasPrefix(strings.ToUpper(mode[i+1:]), "BOM|") + } + return false +} + // modeBase strips a ":enc" encoding suffix (e.g. "w:UTF-16LE") from a mode string. func modeBase(mode string) string { if i := strings.IndexByte(mode, ':'); i >= 0 { diff --git a/internal/vm/io_descriptors.go b/internal/vm/io_descriptors.go index a61fdcf6..7b0c3910 100644 --- a/internal/vm/io_descriptors.go +++ b/internal/vm/io_descriptors.go @@ -49,13 +49,12 @@ func defIOReadExtra(cls *RClass) { // One character is a full character of the stream's external encoding — a // multi-byte EUC-JP/UTF-8 char, not a single byte — which is then transcoded // to the internal encoding when one is set (io.c io_getc → read_all path). + // decodeCharFrom never reports more bytes than remain (an incomplete lead + // yields 0, taken as a one-byte character), so the slice below is in bounds. ext, _ := vm.ioReadEnc(o) _, sz, _, _ := vm.decodeCharFrom(o.buf[o.pos:], ext) if sz < 1 { - sz = 1 // an invalid lead byte is returned as a one-byte character - } - if o.pos+sz > len(o.buf) { - sz = len(o.buf) - o.pos + sz = 1 // an invalid or truncated lead byte is a one-byte character } charBytes := o.buf[o.pos : o.pos+sz] o.pos += sz @@ -339,10 +338,8 @@ func defIOSeekable(cls *RClass) { return object.NilV } o.extEnc = enc - if e, ok := vm.findEncoding(enc); ok { - return e - } - return object.NilV + e, _ := vm.findEncoding(enc) // every BOM name is a registered encoding + return e }) } @@ -352,26 +349,29 @@ func defIOSeekable(cls *RClass) { // truncated BOM leaves the cursor where it was, and "\xFF\xFE" followed by two // NUL bytes is UTF-32LE while "\xFF\xFE" alone is UTF-16LE. func ioStripBOM(o *IOObj) string { - p := o.buf[o.pos:] + n, enc := detectBOM(o.buf[o.pos:]) + o.pos += n + return enc +} + +// detectBOM reports the byte length and encoding name of a leading Unicode +// byte-order mark in p (0, "" when there is none). "\xFF\xFE" followed by two +// NUL bytes is UTF-32LE, while "\xFF\xFE" alone is UTF-16LE (io.c io_strip_bom). +func detectBOM(p []byte) (int, string) { switch { case len(p) >= 3 && p[0] == 0xEF && p[1] == 0xBB && p[2] == 0xBF: - o.pos += 3 - return "UTF-8" + return 3, "UTF-8" case len(p) >= 2 && p[0] == 0xFF && p[1] == 0xFE: if len(p) >= 4 && p[2] == 0x00 && p[3] == 0x00 { - o.pos += 4 - return "UTF-32LE" + return 4, "UTF-32LE" } - o.pos += 2 - return "UTF-16LE" + return 2, "UTF-16LE" case len(p) >= 2 && p[0] == 0xFE && p[1] == 0xFF: - o.pos += 2 - return "UTF-16BE" + return 2, "UTF-16BE" case len(p) >= 4 && p[0] == 0x00 && p[1] == 0x00 && p[2] == 0xFE && p[3] == 0xFF: - o.pos += 4 - return "UTF-32BE" + return 4, "UTF-32BE" } - return "" + return 0, "" } // ioUnget inserts p immediately before the cursor (leaving the cursor on the diff --git a/internal/vm/io_wave20_test.go b/internal/vm/io_wave20_test.go new file mode 100644 index 00000000..5c43698c --- /dev/null +++ b/internal/vm/io_wave20_test.go @@ -0,0 +1,253 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm_test + +import ( + "fmt" + "path/filepath" + "testing" +) + +// eucFixture writes the EUC-JP bytes of "ありがとう\n" to a temp file and returns +// a Ruby prelude binding @p to its (forward-slash) path. The bytes are written +// raw so the test does not depend on a UTF-8→EUC-JP encoder. +func eucFixture(t *testing.T) (path, prelude string) { + t.Helper() + p := filepath.ToSlash(filepath.Join(t.TempDir(), "euc.txt")) + // A4A2 A4EA A4AC A4C8 A4A6 = ありがとう in EUC-JP, then a newline. + return p, fmt.Sprintf("@p=%q; File.binwrite(@p, \"\\xA4\\xA2\\xA4\\xEA\\xA4\\xAC\\xA4\\xC8\\xA4\\xA6\\n\"); ", p) +} + +// TestIOWave20FileOpenHashOptions covers File.open / File.new / Kernel#open +// treating a trailing Hash as options (mode / :encoding / :external_encoding / +// :internal_encoding) rather than a mode String — io.c rb_scan_args "12:" + +// rb_io_extract_modeenc. Asserted against MRI Ruby 4.0.5. +func TestIOWave20FileOpenHashOptions(t *testing.T) { + _, pre := eucFixture(t) + cases := []struct{ src, want string }{ + // String mode with an "ext:int" suffix records both encodings. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); p [f.external_encoding.name, f.internal_encoding.name]`, + "[\"EUC-JP\", \"UTF-8\"]\n"}, + // mode: option (a trailing Hash) is options, not the mode String. + {pre + `f=File.open(@p, mode: "r:euc-jp:utf-8"); p [f.external_encoding.name, f.internal_encoding.name]`, + "[\"EUC-JP\", \"UTF-8\"]\n"}, + // :external_encoding / :internal_encoding options. + {pre + `f=File.open(@p, mode: "r", external_encoding: "euc-jp", internal_encoding: "utf-8"); p f.internal_encoding.name`, + "\"UTF-8\"\n"}, + // :encoding "ext:int" option. + {pre + `f=File.open(@p, mode: "r", encoding: "euc-jp:utf-8"); p f.external_encoding.name`, + "\"EUC-JP\"\n"}, + // File.new goes through the same resolver. + {pre + `f=File.new(@p, mode: "r:euc-jp"); p f.external_encoding.name`, "\"EUC-JP\"\n"}, + // Kernel#open routes to File.open. + {pre + `f=open(@p, mode: "r:euc-jp"); p f.external_encoding.name`, "\"EUC-JP\"\n"}, + // Integer mode via File::RDONLY, and a to_int object. + {pre + `p File.open(@p, File::RDONLY).read.bytesize`, "11\n"}, + {pre + `o=Object.new; def o.to_int; File::RDONLY; end; p File.open(@p, o).read.bytesize`, "11\n"}, + // to_str object as the mode. + {pre + `o=Object.new; def o.to_str; "rb"; end; p File.open(@p, o).binmode?`, "true\n"}, + // nil positional mode falls back to the :mode option. + {pre + `f=File.open(@p, nil, mode: "r:euc-jp"); p f.external_encoding.name`, "\"EUC-JP\"\n"}, + // nil :mode option leaves the default read mode. + {pre + `f=File.open(@p, mode: nil); p f.external_encoding.name`, "\"UTF-8\"\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } + // A bad (non-String, non-Integer, non-coercible) mode raises TypeError. + if cls, msg := evalErr(t, pre+`File.open(@p, Object.new)`); cls != "TypeError" || msg != "no implicit conversion of Object into String" { + t.Errorf("bad mode: got %s: %q", cls, msg) + } + // A Hash-only call (no path) is a wrong-number-of-arguments error. + if cls, _ := evalErr(t, `File.open(mode: "r")`); cls != "ArgumentError" { + t.Errorf("hash-only File.open: got %s", cls) + } +} + +// TestIOWave20ReadTranscoding covers IO#read honouring the external/internal +// encoding: a full read transcodes external→internal and tags the result; a +// length read stays BINARY; a passed buffer is retagged only on a full read +// (io.c io_read / io_enc_str). Asserted against MRI Ruby 4.0.5. +func TestIOWave20ReadTranscoding(t *testing.T) { + _, pre := eucFixture(t) + cases := []struct{ src, want string }{ + // Whole read transcodes EUC-JP→UTF-8. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); s=f.read; p [s, s.encoding.name]`, + "[\"ありがとう\\n\", \"UTF-8\"]\n"}, + // External-only: no transcoding, tagged EUC-JP. + {pre + `f=File.open(@p,"r:euc-jp"); p f.read.encoding.name`, "\"EUC-JP\"\n"}, + // read(size) returns BINARY, position advances. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); p [f.read(2).bytes, f.read(2).encoding.name]`, + "[[164, 162], \"ASCII-8BIT\"]\n"}, + // read(nil, buf): buffer is retagged to the internal encoding and equal to result. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); b="".dup.force_encoding("ISO-8859-1"); r=f.read(nil,b); p [b.equal?(r), b.encoding.name]`, + "[true, \"UTF-8\"]\n"}, + // read(size, buf): buffer bytes set, its encoding left unchanged. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); b="".dup.force_encoding("ISO-8859-1"); f.read(2,b); p [b.bytes, b.encoding.name]`, + "[[164, 162], \"ISO-8859-1\"]\n"}, + // read(size) at EOF clears a passed buffer, leaving its encoding, returns nil. + {pre + `f=File.open(@p,"r:euc-jp"); f.read; b="abc".dup.force_encoding("ISO-8859-1"); r=f.read(1,b); p [r, b.size, b.encoding.name]`, + "[nil, 0, \"ISO-8859-1\"]\n"}, + // read(size) at EOF with no buffer is nil; read(0) is "". + {pre + `f=File.open(@p,"r:euc-jp"); f.read; p [f.read(1), f.read(0)]`, "[nil, \"\"]\n"}, + // Pure-ASCII content is retagged to the internal encoding without a converter. + {`File.write(@q="` + filepathTmp(t, "ascii.txt") + `", "line"); f=File.open(@q, mode: "r", external_encoding: "IBM866", internal_encoding: "utf-8"); s=f.read; p [s, s.encoding.name]`, + "[\"line\", \"UTF-8\"]\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } + // A frozen output buffer on a full read is a FrozenError (io.c rb_str_modify). + if cls, msg := evalErr(t, pre+`f=File.open(@p,"r:euc-jp:utf-8"); f.read(nil, "x".freeze)`); cls != "FrozenError" || msg != `can't modify frozen String: "x"` { + t.Errorf("frozen read buffer: got %s: %q", cls, msg) + } +} + +// filepathTmp returns a forward-slash temp path for name in a fresh temp dir. +func filepathTmp(t *testing.T, name string) string { + t.Helper() + return filepath.ToSlash(filepath.Join(t.TempDir(), name)) +} + +// TestIOWave20Readchar covers IO#readchar decoding one character of the external +// encoding and transcoding to the internal one, plus the EOFError path (io.c +// io_getc). Asserted against MRI Ruby 4.0.5. +func TestIOWave20Readchar(t *testing.T) { + _, pre := eucFixture(t) + cases := []struct{ src, want string }{ + // One EUC-JP character transcoded to UTF-8. + {pre + `f=File.open(@p,"r:euc-jp:utf-8"); c=f.readchar; p [c, c.encoding.name]`, "[\"あ\", \"UTF-8\"]\n"}, + // External-only: the two raw EUC-JP bytes tagged EUC-JP. + {pre + `f=File.open(@p,"r:euc-jp"); c=f.readchar; p [c.bytes, c.encoding.name]`, "[[164, 162], \"EUC-JP\"]\n"}, + // A plain UTF-8 file yields one UTF-8 character. + {`File.write(@p="` + filepathTmp(t, "u.txt") + `", "aé"); f=File.open(@p); p [f.readchar, f.readchar]`, "[\"a\", \"é\"]\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } + // readchar at EOF raises EOFError. + if cls, msg := evalErr(t, pre+`f=File.open(@p,"r:euc-jp"); f.read; f.readchar`); cls != "EOFError" || msg != "end of file reached" { + t.Errorf("readchar EOF: got %s: %q", cls, msg) + } + // An invalid lead byte is returned as a one-byte character. + if got := eval(t, `File.binwrite(@p="`+filepathTmp(t, "bad.txt")+`", "\xFF"); f=File.open(@p, "r:utf-8"); c=f.readchar; p c.bytes`); got != "[255]\n" { + t.Errorf("readchar invalid lead: got %q", got) + } + // A truncated multi-byte lead at EOF is also a one-byte character (sz<1 branch). + if got := eval(t, `File.binwrite(@p="`+filepathTmp(t, "tr.txt")+`", "\xE3"); f=File.open(@p, "r:utf-8"); p f.readchar.bytes`); got != "[227]\n" { + t.Errorf("readchar truncated lead: got %q", got) + } +} + +// TestIOWave20GetsEncoding covers the gets family tagging/transcoding each line +// per the stream's external/internal encoding, including the BINARY suppression +// of the internal encoding (io.c rb_io_getline_1). Asserted against MRI 4.0.5. +func TestIOWave20GetsEncoding(t *testing.T) { + q := filepathTmp(t, "line.txt") + pre := fmt.Sprintf("@p=%q; File.write(@p, \"line\"); ", q) + cases := []struct{ src, want string }{ + {pre + `p File.open(@p,"r").gets.encoding.name`, "\"UTF-8\"\n"}, + {pre + `f=File.open(@p,"r"); f.set_encoding("US-ASCII"); p f.gets.encoding.name`, "\"US-ASCII\"\n"}, + // default_internal transcodes an ASCII line without a converter. + {pre + `Encoding.default_internal="US-ASCII"; f=File.open(@p,"r"); e=f.gets.encoding.name; Encoding.default_internal=nil; p e`, "\"US-ASCII\"\n"}, + // A BINARY external encoding suppresses the internal one. + {pre + `f=File.open(@p,"r"); f.set_encoding("BINARY","UTF-8"); p f.gets.encoding.name`, "\"ASCII-8BIT\"\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } +} + +// TestIOWave20SetEncodingByBOM covers IO#set_encoding_by_bom (io.c +// rb_io_set_encoding_by_bom + io_strip_bom): each BOM's detection and encoding, +// the truncated / absent BOM nil paths, the not-readable nil, and the binmode / +// encoding-already-set / conversion-set ArgumentErrors and closed IOError. +// Asserted against MRI Ruby 4.0.5. +func TestIOWave20SetEncodingByBOM(t *testing.T) { + p := filepathTmp(t, "bom.txt") + open := fmt.Sprintf("@p=%q; File.binwrite(@p,\"\"); touch=->(s){File.binwrite(@p,s)}; ", p) + // Each BOM is detected, stripped, and names the external encoding. + boms := []struct{ bytes, enc, rest string }{ + {`\xEF\xBB\xBFabc`, "UTF-8", "abc"}, + {`\xFF\xFEabc`, "UTF-16LE", "abc"}, + {`\xFE\xFFabcd`, "UTF-16BE", "abcd"}, + {`\xFF\xFE\x00\x00abc`, "UTF-32LE", "abc"}, + {`\x00\x00\xFE\xFFabcd`, "UTF-32BE", "abcd"}, + } + for _, b := range boms { + src := open + fmt.Sprintf("io=File.open(@p,\"rb\"); touch.(\"%s\"); r=io.set_encoding_by_bom; p [r.name, io.external_encoding.name, io.read.b]", b.bytes) + want := fmt.Sprintf("[%q, %q, %q]\n", b.enc, b.enc, b.rest) + if got := eval(t, src); got != want { + t.Errorf("BOM %s: got=%q want=%q", b.enc, got, want) + } + } + // FF FE followed by a single NUL is UTF-16LE (not the incomplete UTF-32LE). + if got := eval(t, open+`io=File.open(@p,"rb"); touch.("\xFF\xFE\x00"); r=io.set_encoding_by_bom; p [r.name, io.read.b]`); got != "[\"UTF-16LE\", \"\\x00\"]\n" { + t.Errorf("FFFE00: got %q", got) + } + // No / truncated / absent BOM returns nil and leaves ASCII-8BIT + all bytes. + nilCases := []struct{ bytes, rest string }{ + {`\xEF`, "\\xEF"}, {`\xEF\xBB`, "\\xEF\\xBB"}, {`\xFE`, "\\xFE"}, + {`\xFF`, "\\xFF"}, {`\x00`, "\\x00"}, {`\x00\x00\xFE`, "\\x00\\x00\\xFE"}, + {`abc`, "abc"}, {``, ""}, + } + for _, c := range nilCases { + src := open + fmt.Sprintf("io=File.open(@p,\"rb\"); touch.(\"%s\"); r=io.set_encoding_by_bom; p [r, io.external_encoding.name, io.read.b]", c.bytes) + want := "[nil, \"ASCII-8BIT\", \"" + c.rest + "\"]\n" + if got := eval(t, src); got != want { + t.Errorf("nil BOM %q: got=%q want=%q", c.bytes, got, want) + } + } + // A write-only stream is not readable: nil, encoding untouched. + if got := eval(t, open+`touch.(""); io=File.open(@p,"wb"); p [io.set_encoding_by_bom, io.external_encoding.name]`); got != "[nil, \"ASCII-8BIT\"]\n" { + t.Errorf("wb set_encoding_by_bom: got %q", got) + } + // Error paths. + errCases := []struct{ src, class, msg string }{ + {open + `touch.(""); File.open(@p,"r").set_encoding_by_bom`, "ArgumentError", "ASCII incompatible encoding needs binmode"}, + {open + `touch.(""); io=File.open(@p,"rb"); io.set_encoding("utf-8"); io.set_encoding_by_bom`, "ArgumentError", "encoding is set to UTF-8 already"}, + {open + `touch.(""); io=File.open(@p,"rb"); io.set_encoding("utf-8","utf-16be"); io.set_encoding_by_bom`, "ArgumentError", "encoding conversion is set"}, + {open + `touch.(""); io=File.open(@p,"rb"); io.close; io.set_encoding_by_bom`, "IOError", "closed stream"}, + } + for _, c := range errCases { + cls, msg := evalErr(t, c.src) + if cls != c.class || msg != c.msg { + t.Errorf("src=%q\n got=%s: %q\nwant=%s: %q", c.src, cls, msg, c.class, c.msg) + } + } +} + +// TestIOWave20ReadBOMMode covers IO.read / File.read with a "rb:BOM|enc" mode +// stripping a leading BOM (io.c) and modeHasBOM's ':'-less path. Asserted +// against MRI Ruby 4.0.5. +func TestIOWave20ReadBOMMode(t *testing.T) { + p := filepathTmp(t, "r.txt") + pre := fmt.Sprintf("@p=%q; ", p) + cases := []struct{ src, want string }{ + // UTF-8 BOM stripped. + {pre + `File.binwrite(@p,"\xEF\xBB\xBFhi"); p File.read(@p, mode: "rb:BOM|utf-16le").b`, "\"hi\"\n"}, + // UTF-16LE BOM stripped; remaining bytes kept. + {pre + `File.binwrite(@p,"\xFF\xFEhi"); p File.read(@p, mode: "rb:BOM|utf-8").b`, "\"hi\"\n"}, + // No BOM present: BOM mode is a no-op on the bytes. + {pre + `File.binwrite(@p,"hi"); p File.read(@p, mode: "rb:BOM|utf-8").b`, "\"hi\"\n"}, + // A plain mode (no ':') does not strip. + {pre + `File.binwrite(@p,"\xEF\xBB\xBFhi"); p File.read(@p, mode: "rb").b`, "\"\\xEF\\xBB\\xBFhi\"\n"}, + // BOM strip only applies at offset 0: with an offset it is not consulted. + {pre + `File.binwrite(@p,"\xEF\xBB\xBFhi"); p File.read(@p, 2, 1, mode: "rb:BOM|utf-8").b`, "\"\\xBB\\xBF\"\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } +} From 72e460f5aa75f4789709485a58b78a5036404e9c Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 8 Sep 2026 12:34:22 +0200 Subject: [PATCH 3/3] io: route IO.sysopen mode through vmodeString, drop now-unused fileMode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File.open/File.new/Kernel#open no longer call fileMode (they resolve the mode through openFileArgs/vmodeString), leaving IO.sysopen its only caller — and sysopen always passes a mode, so fileMode's default branch became dead code. Inline the resolution into sysopen via vmodeString (which also coerces an Integer flag set or a #to_str mode object, unlike the old strArg) and delete fileMode. Also drop File.open's redundant zero-arg guard: openFileArgs already rejects a missing path with the same ArgumentError. Keeps the touched files at 100% statement coverage. Co-Authored-By: Claude Opus 4.8 --- internal/vm/io.go | 18 +----------------- internal/vm/io_class.go | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/internal/vm/io.go b/internal/vm/io.go index be094362..2a978769 100644 --- a/internal/vm/io.go +++ b/internal/vm/io.go @@ -410,10 +410,7 @@ func (vm *VM) registerIO() { cFile := vm.consts["File"].(*RClass) cFile.super = cIO // File < IO, inheriting the read+write protocol; is_a?(IO) holds cFile.smethods["open"] = &Method{name: "open", owner: cFile, native: func(vm *VM, _ object.Value, args []object.Value, blk *Proc) object.Value { - if len(args) == 0 { - raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") - } - o := vm.openFileArgs(cFile, args) + o := vm.openFileArgs(cFile, args) // openFileArgs rejects a missing path if blk != nil { defer ioFlushClose(o) return vm.callBlock(blk, []object.Value{o}) @@ -527,19 +524,6 @@ func flagsToMode(flags int64) string { } } -// fileMode returns the access mode argument of File.open (default "r"). The mode -// may be a string ("w", "r+", ...) or an integer bit-OR of File::Constants flags -// (e.g. File::RDWR | File::CREAT | File::EXCL); a trailing opts Hash is ignored. -func fileMode(args []object.Value) string { - if len(args) > 1 { - if i, ok := args[1].(object.Integer); ok { - return flagsToMode(int64(i)) - } - return strArg(args[1]) - } - return "r" -} - // stringIOModeVal resolves a StringIO mode argument to either an Integer flag set // or a String access mode: Integer and String are taken directly, and any other // object is converted via #to_str (raising TypeError otherwise). Keeping the diff --git a/internal/vm/io_class.go b/internal/vm/io_class.go index d65b2c34..919a2538 100644 --- a/internal/vm/io_class.go +++ b/internal/vm/io_class.go @@ -84,7 +84,7 @@ func (vm *VM) registerIOClassMethods(cIO, cFile *RClass) { path := pathArg(vm, args[0]) mode := "r" if len(args) >= 2 && !object.IsNil(args[1]) { - mode = fileMode(args[:2]) + mode = vm.vmodeString(args[1]) // Integer flag set or fopen-style String } o := openFileIO(cFile, path, mode) vm.nextFd++