Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 76 additions & 9 deletions internal/vm/enumerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ type Enumerator struct {
peeked bool
ended bool
finish object.Value
// feedVal (when feedSet) is the value #feed queued for the source's next
// `yield` to return during external iteration; it is consumed the next time the
// driving fiber resumes past a yield. #feed refuses a second value before the
// first is consumed (TypeError), and #rewind discards a pending one.
feedVal object.Value
feedSet bool
}

// forPull returns a copy of e carrying its definition and none of its
Expand All @@ -86,8 +92,12 @@ func (e *Enumerator) forPull() *Enumerator {
}

// yielder is the object passed to an Enumerator.new generator block; `y << v`
// and `y.yield(v)` feed values into the enumeration.
type yielder struct{ emit func(args []object.Value) }
// and `y.yield(v)` feed values into the enumeration. emit returns the value the
// downstream consumer produced for this yield — the #feed value during external
// iteration — which `y.yield` surfaces as its own result (MRI's Yielder#yield).
type yielder struct {
emit func(args []object.Value) object.Value
}

func (y *yielder) ToS() string { return "#<Enumerator::Yielder>" }
func (y *yielder) Inspect() string { return y.ToS() }
Expand Down Expand Up @@ -230,12 +240,22 @@ func (vm *VM) registerEnumerator() {
vm.cYielder.consts = vm.cEnumerator.consts // (scope is cosmetic; share the map)
vm.cEnumerator.consts["Yielder"] = vm.cYielder
vm.cYielder.define("<<", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value {
if len(args) != 1 {
raise("ArgumentError", "wrong number of arguments (given %d, expected 1)", len(args))
}
self.(*yielder).emit(args)
return self // << chains
})
vm.cYielder.define("yield", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value {
self.(*yielder).emit(args)
return object.NilV
return self.(*yielder).emit(args)
})
// &yielder passes the yielder as a block: its #to_proc calls #<< (so
// `str.each_line(&y)` feeds each line into the enumeration).
vm.cYielder.define("to_proc", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
y := self.(*yielder)
return &Proc{native: func(vm *VM, args []object.Value) object.Value {
return y.emit(args)
}}
})
// Enumerator.new(size = nil) { |y| … } builds a generator-block enumerator. An
// optional leading argument sets #size (an Integer/nil returned verbatim, or a
Expand Down Expand Up @@ -279,8 +299,34 @@ func (vm *VM) registerEnumerator() {
}

d := func(name string, fn NativeFn) { vm.cEnumerator.define(name, fn) }
d("each", func(vm *VM, self object.Value, _ []object.Value, blk *Proc) object.Value {
// #inspect / #to_s: a live Enumerator or Lazy renders through its Go Inspect;
// a bare instance from Class#allocate (never #initialize-d) reads
// "#<ClassName: uninitialized>", as MRI shows for an uninitialized enumerator.
inspectFn := func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
switch v := self.(type) {
case *Enumerator:
return object.NewString(v.Inspect())
case *LazyEnum:
return object.NewString(v.Inspect())
default:
return object.NewString("#<" + vm.classOf(self).name + ": uninitialized>")
}
}
d("inspect", inspectFn)
d("to_s", inspectFn)
d("each", func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value {
e := self.(*Enumerator)
if len(args) > 0 {
// #each(*extra) appends the extra arguments to the receiver's #each call:
// a fresh enumerator with the wider argument list, run (or returned) here.
ne := *e
ne.args = append(append([]object.Value{}, e.args...), args...)
ne.extFiber, ne.peeked, ne.ended = nil, false, false
if blk == nil {
return &ne
}
return vm.enumRunEach(&ne, blk)
}
if blk == nil {
return e
}
Expand Down Expand Up @@ -332,9 +378,21 @@ func (vm *VM) registerEnumerator() {
d("peek_values", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
return vm.enumPeekRaw(self.(*Enumerator))
})
d("feed", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value {
e := self.(*Enumerator)
if e.feedSet {
raise("TypeError", "feed value already set")
}
var v object.Value = object.NilV
if len(args) > 0 {
v = args[0]
}
e.feedVal, e.feedSet = v, true
return object.NilV
})
d("rewind", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value {
e := self.(*Enumerator)
e.extFiber, e.peeked, e.ended = nil, false, false
e.extFiber, e.peeked, e.ended, e.feedSet = nil, false, false, false
if e.isChain {
// Rewind the parts that were iterated, in reverse, and forget them.
for i := len(e.chainParts) - 1; i >= 0; i-- {
Expand Down Expand Up @@ -378,7 +436,10 @@ func (vm *VM) registerEnumerator() {
return vm.enumRunEach(e, wrapper)
}
d("with_index", withIndex)
d("each_with_index", func(vm *VM, self object.Value, _ []object.Value, blk *Proc) object.Value {
d("each_with_index", func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value {
if len(args) > 0 {
raise("ArgumentError", "wrong number of arguments (given %d, expected 0)", len(args))
}
return withIndex(vm, self, nil, blk) // each_with_index ignores any offset
})
// with_object is MRI's alias of each_with_object; share the one Method so
Expand Down Expand Up @@ -557,8 +618,8 @@ func (vm *VM) enumRunEach(e *Enumerator, blk *Proc) object.Value {
case e.produceBlk != nil:
return vm.enumRunProduce(e, blk)
case e.block != nil:
return vm.callBlock(e.block, []object.Value{&yielder{emit: func(args []object.Value) {
vm.callBlock(blk, args)
return vm.callBlock(e.block, []object.Value{&yielder{emit: func(args []object.Value) object.Value {
return vm.callBlock(blk, args)
}}})
default:
return vm.send(e.recv, e.meth, e.args, blk)
Expand Down Expand Up @@ -613,6 +674,12 @@ func (vm *VM) enumFiber(e *Enumerator) *Fiber {
collect := &Proc{native: func(vm *VM, args []object.Value) object.Value {
wrapper := object.NewArrayFromSlice(append([]object.Value{}, args...))
vm.fiberYield([]object.Value{wrapper})
// On resume, hand the source's `yield` any value #feed queued.
if e.feedSet {
fv := e.feedVal
e.feedVal, e.feedSet = nil, false
return fv
}
return object.NilV
}}
return vm.enumRunEach(e, collect)
Expand Down
226 changes: 226 additions & 0 deletions internal/vm/enumerator_residuals_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,226 @@
package vm_test

import (
"strings"
"testing"
)

// TestEnumeratorResiduals covers the Enumerator + Enumerator::Lazy conformance
// residuals: #feed, Yielder#to_proc/#<< arity, #each with appended arguments,
// #each_with_index arity, the uninitialized #inspect form, Enumerator::Lazy.new,
// lazy #size propagation, lazy external iteration, lazy #to_enum/#enum_for, the
// method aliases, lazy multi-value block semantics (rb_yield_values2 vs gathered),
// flat_map flattening, lazy #zip validation, lazy #with_index nil offset and lazy
// #take quota/zero-count behaviour — all asserted against MRI Ruby 4.0.
func TestEnumeratorResiduals(t *testing.T) {
cases := []struct{ src, want string }{
// --- #feed: the queued value is what the source's next yield returns ---
{`e = Object.new
def e.each; 3.times { |i| $out << yield }; end
$out = []
en = e.to_enum(:each)
en.feed :a
en.next; en.next; en.next
p $out`, "[:a, nil]\n"},
// #feed called during iteration; #feed returns nil
{`e = Object.new
def e.each; 3.times { $out << yield }; end
$out = []
en = e.to_enum(:each)
en.next
p(en.feed(:a))
en.next; en.next
p $out`, "nil\n[:a, nil]\n"},
// Yielder#yield surfaces the fed value
{`out = []
enum = Enumerator.new { |y| out << y.yield }
enum.next
enum.feed :a
(enum.next rescue nil)
p out`, "[:a]\n"},
// #rewind discards a pending feed
{`e = Object.new
def e.each; 2.times { $out << yield }; end
$out = []
en = e.to_enum(:each)
en.feed :a
en.rewind
en.next; en.next
p $out`, "[nil]\n"},

// --- Yielder#to_proc: &yielder feeds each element in ---
{`e = Enumerator.new { |y| "a\nb".each_line(&y) }
p e.to_a`, "[\"a\\n\", \"b\"]\n"},
// Yielder#yield returns the yielder-consumer value (nil here), #<< chains
{`e = Enumerator.new { |y| (y << 1) << 2 }
p e.to_a`, "[1, 2]\n"},

// --- Enumerator#each with appended arguments ---
{`o = Object.new
def o.each(*a); yield a; end
e = o.to_enum(:each, :x, :y)
p e.each(:z).to_a`, "[[:x, :y, :z]]\n"},
// each(*args) with a block forwards through
{`o = Object.new
def o.each(*a); yield a; :ret; end
e = o.to_enum(:each, :x)
p(e.each(:y) { |v| $seen = v })
p $seen`, ":ret\n[:x, :y]\n"},

// --- uninitialized #inspect ---
{`p Enumerator.allocate.inspect`, "\"#<Enumerator: uninitialized>\"\n"},
{`p Enumerator::Lazy.allocate.inspect`, "\"#<Enumerator::Lazy: uninitialized>\"\n"},
// a live enumerator still inspects normally
{`p [1, 2].each.inspect`, "\"#<Enumerator: [1, 2]:each>\"\n"},
// a live lazy inspects through its own Inspect
{`p (1..3).lazy.map { |x| x }.inspect`, "\"#<Enumerator::Lazy: 1..3:map>\"\n"},

// --- Enumerator::Lazy.new ---
{`r = Object.new
def r.each; yield 0; yield 1; yield 2; end
p Enumerator::Lazy.new(r) { |y, *v| y.<<(*v) }.first(2)`, "[0, 1]\n"},
{`p Enumerator::Lazy.new(Object.new, 100) {}.map {}.size`, "100\n"},
{`p Enumerator::Lazy.new(Object.new, 100) {}.select { true }.size`, "nil\n"},
{`p Enumerator::Lazy.new(Object.new, nil) {}.size`, "nil\n"},
{`p Enumerator::Lazy.new(Object.new, -> { 7 }) {}.size`, "7\n"},

// --- lazy #size across the op chain ---
{`p (1..100).lazy.map { |x| x }.size`, "100\n"},
{`p (1..100).lazy.with_index.size`, "100\n"},
{`p (1..100).lazy.zip([1]).size`, "100\n"},
{`p (1..100).lazy.reject { |x| x }.size`, "nil\n"},
{`p (1..100).lazy.take(20).size`, "20\n"},
{`p (1..100).lazy.take(200).size`, "100\n"},
{`p (1..100).lazy.drop(20).size`, "80\n"},
{`p (1..100).lazy.drop(200).size`, "0\n"},
{`p (1..Float::INFINITY).lazy.take(20).size`, "20\n"},
{`p (1..Float::INFINITY).lazy.drop(20).size`, "Infinity\n"},
{`p (1..Float::INFINITY).lazy.map { |x| x }.size`, "Infinity\n"},
// a source with no #size yields nil
{`o = Object.new
def o.each; yield 1; end
o.extend(Enumerable)
p o.lazy.map { |x| x }.size`, "nil\n"},

// --- lazy external iteration (no over-evaluation) ---
{`n = 0
e = [1, 2, 3].lazy.select { n += 1; true }
p n
e.peek; e.peek
p n
e.next
p n`, "0\n1\n1\n"},
{`e = (1..Float::INFINITY).lazy.map { |x| x * 2 }
p [e.next, e.next, e.next]`, "[2, 4, 6]\n"},
{`e = [1, 2].lazy.map { |x| x }
e.next; e.rewind
p e.next`, "1\n"},
{`e = [10, 20].lazy.map { |x| x }
p e.next_values
p e.peek_values`, "[10]\n[20]\n"},

// --- lazy #to_enum / #enum_for return a distinct Lazy ---
{`l = (1..3).lazy
p l.to_enum.class`, "Enumerator::Lazy\n"},
{`l = (1..3).lazy
p l.to_enum.equal?(l)`, "false\n"},
{`p Enumerator::Lazy.new(Object.new, 100) {}.to_enum { 30 }.size`, "30\n"},
{`p Enumerator::Lazy.new(Object.new, 100) {}.to_enum.size`, "nil\n"},
// to_enum(meth, ...) drives self.meth lazily when forced
{`p (1..5).lazy.to_enum(:each).first(3)`, "[1, 2, 3]\n"},
{`p (1..5).lazy.to_enum.first(3)`, "[1, 2, 3]\n"},

// --- aliases share the method object ---
{`p Enumerator::Lazy.instance_method(:collect) == Enumerator::Lazy.instance_method(:map)`, "true\n"},
{`p Enumerator::Lazy.instance_method(:filter) == Enumerator::Lazy.instance_method(:select)`, "true\n"},
{`p Enumerator::Lazy.instance_method(:find_all) == Enumerator::Lazy.instance_method(:select)`, "true\n"},
{`p Enumerator::Lazy.instance_method(:collect_concat) == Enumerator::Lazy.instance_method(:flat_map)`, "true\n"},
{`p Enumerator::Lazy.instance_method(:enum_for) == Enumerator::Lazy.instance_method(:to_enum)`, "true\n"},

// --- multi-value block semantics (rb_yield_values2 vs gathered) ---
// map/flat_map/filter_map/take_while/drop_while see the first value
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.map { |v| v }.force`, "[0, 2]\n"},
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.flat_map { |v| [v] }.force`, "[0, 2]\n"},
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.take_while { |v| true }.force`, "[[0, 1], [2, 3]]\n"},
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.drop_while { |v| false }.force`, "[[0, 1], [2, 3]]\n"},
// select/reject/uniq see the gathered array
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
s = []
o.to_enum.lazy.select { |v| s << v; true }.force
p s`, "[[0, 1], [2, 3]]\n"},
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.uniq.force`, "[[0, 1], [2, 3]]\n"},
// gathered multi-value survives a filter into a downstream map (sees first)
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.select { true }.map { |v| v }.force`, "[0, 2]\n"},
// with_index gathers the value, no-block pairs it
{`o = Object.new
def o.each; yield 0, 1; yield 2, 3; end
p o.to_enum.lazy.with_index.force`, "[[[0, 1], 0], [[2, 3], 1]]\n"},

// --- flat_map flattening rules ---
{`p [1, 2].lazy.flat_map { |n| [n, -n] }.force`, "[1, -1, 2, -2]\n"},
{`p [1, 2].lazy.flat_map { |n| n.to_s.each_char.lazy }.force`, "[\"1\", \"2\"]\n"},
{`p [1, 2].lazy.flat_map { |n| n.to_s.each_char }.first(2).all? { |o| o.instance_of?(Enumerator) }`, "true\n"},

// --- with_index nil offset starts at 0 ---
{`p [10, 20].lazy.with_index(nil).force`, "[[10, 0], [20, 1]]\n"},
{`p [10, 20].lazy.with_index(5).force`, "[[10, 5], [20, 6]]\n"},

// --- take quota / zero-count ---
{`$out = []
o = Object.new
def o.each; $out << :before; yield 0; $out << :after; end
o.to_enum.lazy.take(1).force
p $out`, "[:before]\n"},
{`$out = []
o = Object.new
def o.each; $out << :before; yield 0; end
o.to_enum.lazy.take(0).force
p $out`, "[]\n"},
{`p (1..Float::INFINITY).lazy.map { |x| x }.take(3).force`, "[1, 2, 3]\n"},
// take followed by an early-stopping terminal unwinds through take
{`p (1..10).lazy.take(5).first(2)`, "[1, 2]\n"},

// --- eager/force still work ---
{`p [1, 2, 3].lazy.map { |x| x * 2 }.to_a`, "[2, 4, 6]\n"},
// a lazy over a Chain enumerator pulls it fresh (forPull rewinds a chain)
{`p (1..2).each.chain([3, 4]).lazy.map { |x| x }.force`, "[1, 2, 3, 4]\n"},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want {
t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want)
}
}

errs := []struct{ src, want string }{
// #feed twice without advancing raises TypeError
{`e = [1, 2].each; e.feed :a; e.feed :b`, "TypeError"},
// Yielder#<< with more than one argument
{`Enumerator.new { |y| y.<<(1, 2) }.to_a`, "wrong number of arguments (given 2, expected 1)"},
// each_with_index rejects extra arguments
{`[1].to_enum.each_with_index(:x)`, "ArgumentError"},
// Enumerator::Lazy.new without a block
{`Enumerator::Lazy.new(Object.new)`, "ArgumentError"},
// Enumerator::Lazy.new without a source
{`Enumerator::Lazy.new { |y| }`, "ArgumentError"},
// lazy zip with a non-list argument
{`[1, 2].lazy.zip([], Object.new)`, "TypeError"},
}
for _, c := range errs {
if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("src=%q got=%v want containing %q", c.src, err, c.want)
}
}
}
Loading