diff --git a/internal/vm/enumerator.go b/internal/vm/enumerator.go index d2fb5f4f..86d1bc3c 100644 --- a/internal/vm/enumerator.go +++ b/internal/vm/enumerator.go @@ -70,6 +70,12 @@ type Enumerator struct { peeked bool ended bool finish object.Value + // feedVal (when feedSet) is the value #feed queued for the source's next + // `yield` to return during external iteration; it is consumed the next time the + // driving fiber resumes past a yield. #feed refuses a second value before the + // first is consumed (TypeError), and #rewind discards a pending one. + feedVal object.Value + feedSet bool } // forPull returns a copy of e carrying its definition and none of its @@ -86,8 +92,12 @@ func (e *Enumerator) forPull() *Enumerator { } // yielder is the object passed to an Enumerator.new generator block; `y << v` -// and `y.yield(v)` feed values into the enumeration. -type yielder struct{ emit func(args []object.Value) } +// and `y.yield(v)` feed values into the enumeration. emit returns the value the +// downstream consumer produced for this yield — the #feed value during external +// iteration — which `y.yield` surfaces as its own result (MRI's Yielder#yield). +type yielder struct { + emit func(args []object.Value) object.Value +} func (y *yielder) ToS() string { return "#" } func (y *yielder) Inspect() string { return y.ToS() } @@ -230,12 +240,22 @@ func (vm *VM) registerEnumerator() { vm.cYielder.consts = vm.cEnumerator.consts // (scope is cosmetic; share the map) vm.cEnumerator.consts["Yielder"] = vm.cYielder vm.cYielder.define("<<", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + if len(args) != 1 { + raise("ArgumentError", "wrong number of arguments (given %d, expected 1)", len(args)) + } self.(*yielder).emit(args) return self // << chains }) vm.cYielder.define("yield", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { - self.(*yielder).emit(args) - return object.NilV + return self.(*yielder).emit(args) + }) + // &yielder passes the yielder as a block: its #to_proc calls #<< (so + // `str.each_line(&y)` feeds each line into the enumeration). + vm.cYielder.define("to_proc", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + y := self.(*yielder) + return &Proc{native: func(vm *VM, args []object.Value) object.Value { + return y.emit(args) + }} }) // Enumerator.new(size = nil) { |y| … } builds a generator-block enumerator. An // optional leading argument sets #size (an Integer/nil returned verbatim, or a @@ -279,8 +299,34 @@ func (vm *VM) registerEnumerator() { } d := func(name string, fn NativeFn) { vm.cEnumerator.define(name, fn) } - d("each", func(vm *VM, self object.Value, _ []object.Value, blk *Proc) object.Value { + // #inspect / #to_s: a live Enumerator or Lazy renders through its Go Inspect; + // a bare instance from Class#allocate (never #initialize-d) reads + // "#", as MRI shows for an uninitialized enumerator. + inspectFn := func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + switch v := self.(type) { + case *Enumerator: + return object.NewString(v.Inspect()) + case *LazyEnum: + return object.NewString(v.Inspect()) + default: + return object.NewString("#<" + vm.classOf(self).name + ": uninitialized>") + } + } + d("inspect", inspectFn) + d("to_s", inspectFn) + d("each", func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value { e := self.(*Enumerator) + if len(args) > 0 { + // #each(*extra) appends the extra arguments to the receiver's #each call: + // a fresh enumerator with the wider argument list, run (or returned) here. + ne := *e + ne.args = append(append([]object.Value{}, e.args...), args...) + ne.extFiber, ne.peeked, ne.ended = nil, false, false + if blk == nil { + return &ne + } + return vm.enumRunEach(&ne, blk) + } if blk == nil { return e } @@ -332,9 +378,21 @@ func (vm *VM) registerEnumerator() { d("peek_values", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return vm.enumPeekRaw(self.(*Enumerator)) }) + d("feed", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + e := self.(*Enumerator) + if e.feedSet { + raise("TypeError", "feed value already set") + } + var v object.Value = object.NilV + if len(args) > 0 { + v = args[0] + } + e.feedVal, e.feedSet = v, true + return object.NilV + }) d("rewind", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { e := self.(*Enumerator) - e.extFiber, e.peeked, e.ended = nil, false, false + e.extFiber, e.peeked, e.ended, e.feedSet = nil, false, false, false if e.isChain { // Rewind the parts that were iterated, in reverse, and forget them. for i := len(e.chainParts) - 1; i >= 0; i-- { @@ -378,7 +436,10 @@ func (vm *VM) registerEnumerator() { return vm.enumRunEach(e, wrapper) } d("with_index", withIndex) - d("each_with_index", func(vm *VM, self object.Value, _ []object.Value, blk *Proc) object.Value { + d("each_with_index", func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value { + if len(args) > 0 { + raise("ArgumentError", "wrong number of arguments (given %d, expected 0)", len(args)) + } return withIndex(vm, self, nil, blk) // each_with_index ignores any offset }) // with_object is MRI's alias of each_with_object; share the one Method so @@ -557,8 +618,8 @@ func (vm *VM) enumRunEach(e *Enumerator, blk *Proc) object.Value { case e.produceBlk != nil: return vm.enumRunProduce(e, blk) case e.block != nil: - return vm.callBlock(e.block, []object.Value{&yielder{emit: func(args []object.Value) { - vm.callBlock(blk, args) + return vm.callBlock(e.block, []object.Value{&yielder{emit: func(args []object.Value) object.Value { + return vm.callBlock(blk, args) }}}) default: return vm.send(e.recv, e.meth, e.args, blk) @@ -613,6 +674,12 @@ func (vm *VM) enumFiber(e *Enumerator) *Fiber { collect := &Proc{native: func(vm *VM, args []object.Value) object.Value { wrapper := object.NewArrayFromSlice(append([]object.Value{}, args...)) vm.fiberYield([]object.Value{wrapper}) + // On resume, hand the source's `yield` any value #feed queued. + if e.feedSet { + fv := e.feedVal + e.feedVal, e.feedSet = nil, false + return fv + } return object.NilV }} return vm.enumRunEach(e, collect) diff --git a/internal/vm/enumerator_residuals_test.go b/internal/vm/enumerator_residuals_test.go new file mode 100644 index 00000000..2c677d7a --- /dev/null +++ b/internal/vm/enumerator_residuals_test.go @@ -0,0 +1,226 @@ +package vm_test + +import ( + "strings" + "testing" +) + +// TestEnumeratorResiduals covers the Enumerator + Enumerator::Lazy conformance +// residuals: #feed, Yielder#to_proc/#<< arity, #each with appended arguments, +// #each_with_index arity, the uninitialized #inspect form, Enumerator::Lazy.new, +// lazy #size propagation, lazy external iteration, lazy #to_enum/#enum_for, the +// method aliases, lazy multi-value block semantics (rb_yield_values2 vs gathered), +// flat_map flattening, lazy #zip validation, lazy #with_index nil offset and lazy +// #take quota/zero-count behaviour — all asserted against MRI Ruby 4.0. +func TestEnumeratorResiduals(t *testing.T) { + cases := []struct{ src, want string }{ + // --- #feed: the queued value is what the source's next yield returns --- + {`e = Object.new +def e.each; 3.times { |i| $out << yield }; end +$out = [] +en = e.to_enum(:each) +en.feed :a +en.next; en.next; en.next +p $out`, "[:a, nil]\n"}, + // #feed called during iteration; #feed returns nil + {`e = Object.new +def e.each; 3.times { $out << yield }; end +$out = [] +en = e.to_enum(:each) +en.next +p(en.feed(:a)) +en.next; en.next +p $out`, "nil\n[:a, nil]\n"}, + // Yielder#yield surfaces the fed value + {`out = [] +enum = Enumerator.new { |y| out << y.yield } +enum.next +enum.feed :a +(enum.next rescue nil) +p out`, "[:a]\n"}, + // #rewind discards a pending feed + {`e = Object.new +def e.each; 2.times { $out << yield }; end +$out = [] +en = e.to_enum(:each) +en.feed :a +en.rewind +en.next; en.next +p $out`, "[nil]\n"}, + + // --- Yielder#to_proc: &yielder feeds each element in --- + {`e = Enumerator.new { |y| "a\nb".each_line(&y) } +p e.to_a`, "[\"a\\n\", \"b\"]\n"}, + // Yielder#yield returns the yielder-consumer value (nil here), #<< chains + {`e = Enumerator.new { |y| (y << 1) << 2 } +p e.to_a`, "[1, 2]\n"}, + + // --- Enumerator#each with appended arguments --- + {`o = Object.new +def o.each(*a); yield a; end +e = o.to_enum(:each, :x, :y) +p e.each(:z).to_a`, "[[:x, :y, :z]]\n"}, + // each(*args) with a block forwards through + {`o = Object.new +def o.each(*a); yield a; :ret; end +e = o.to_enum(:each, :x) +p(e.each(:y) { |v| $seen = v }) +p $seen`, ":ret\n[:x, :y]\n"}, + + // --- uninitialized #inspect --- + {`p Enumerator.allocate.inspect`, "\"#\"\n"}, + {`p Enumerator::Lazy.allocate.inspect`, "\"#\"\n"}, + // a live enumerator still inspects normally + {`p [1, 2].each.inspect`, "\"#\"\n"}, + // a live lazy inspects through its own Inspect + {`p (1..3).lazy.map { |x| x }.inspect`, "\"#\"\n"}, + + // --- Enumerator::Lazy.new --- + {`r = Object.new +def r.each; yield 0; yield 1; yield 2; end +p Enumerator::Lazy.new(r) { |y, *v| y.<<(*v) }.first(2)`, "[0, 1]\n"}, + {`p Enumerator::Lazy.new(Object.new, 100) {}.map {}.size`, "100\n"}, + {`p Enumerator::Lazy.new(Object.new, 100) {}.select { true }.size`, "nil\n"}, + {`p Enumerator::Lazy.new(Object.new, nil) {}.size`, "nil\n"}, + {`p Enumerator::Lazy.new(Object.new, -> { 7 }) {}.size`, "7\n"}, + + // --- lazy #size across the op chain --- + {`p (1..100).lazy.map { |x| x }.size`, "100\n"}, + {`p (1..100).lazy.with_index.size`, "100\n"}, + {`p (1..100).lazy.zip([1]).size`, "100\n"}, + {`p (1..100).lazy.reject { |x| x }.size`, "nil\n"}, + {`p (1..100).lazy.take(20).size`, "20\n"}, + {`p (1..100).lazy.take(200).size`, "100\n"}, + {`p (1..100).lazy.drop(20).size`, "80\n"}, + {`p (1..100).lazy.drop(200).size`, "0\n"}, + {`p (1..Float::INFINITY).lazy.take(20).size`, "20\n"}, + {`p (1..Float::INFINITY).lazy.drop(20).size`, "Infinity\n"}, + {`p (1..Float::INFINITY).lazy.map { |x| x }.size`, "Infinity\n"}, + // a source with no #size yields nil + {`o = Object.new +def o.each; yield 1; end +o.extend(Enumerable) +p o.lazy.map { |x| x }.size`, "nil\n"}, + + // --- lazy external iteration (no over-evaluation) --- + {`n = 0 +e = [1, 2, 3].lazy.select { n += 1; true } +p n +e.peek; e.peek +p n +e.next +p n`, "0\n1\n1\n"}, + {`e = (1..Float::INFINITY).lazy.map { |x| x * 2 } +p [e.next, e.next, e.next]`, "[2, 4, 6]\n"}, + {`e = [1, 2].lazy.map { |x| x } +e.next; e.rewind +p e.next`, "1\n"}, + {`e = [10, 20].lazy.map { |x| x } +p e.next_values +p e.peek_values`, "[10]\n[20]\n"}, + + // --- lazy #to_enum / #enum_for return a distinct Lazy --- + {`l = (1..3).lazy +p l.to_enum.class`, "Enumerator::Lazy\n"}, + {`l = (1..3).lazy +p l.to_enum.equal?(l)`, "false\n"}, + {`p Enumerator::Lazy.new(Object.new, 100) {}.to_enum { 30 }.size`, "30\n"}, + {`p Enumerator::Lazy.new(Object.new, 100) {}.to_enum.size`, "nil\n"}, + // to_enum(meth, ...) drives self.meth lazily when forced + {`p (1..5).lazy.to_enum(:each).first(3)`, "[1, 2, 3]\n"}, + {`p (1..5).lazy.to_enum.first(3)`, "[1, 2, 3]\n"}, + + // --- aliases share the method object --- + {`p Enumerator::Lazy.instance_method(:collect) == Enumerator::Lazy.instance_method(:map)`, "true\n"}, + {`p Enumerator::Lazy.instance_method(:filter) == Enumerator::Lazy.instance_method(:select)`, "true\n"}, + {`p Enumerator::Lazy.instance_method(:find_all) == Enumerator::Lazy.instance_method(:select)`, "true\n"}, + {`p Enumerator::Lazy.instance_method(:collect_concat) == Enumerator::Lazy.instance_method(:flat_map)`, "true\n"}, + {`p Enumerator::Lazy.instance_method(:enum_for) == Enumerator::Lazy.instance_method(:to_enum)`, "true\n"}, + + // --- multi-value block semantics (rb_yield_values2 vs gathered) --- + // map/flat_map/filter_map/take_while/drop_while see the first value + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.map { |v| v }.force`, "[0, 2]\n"}, + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.flat_map { |v| [v] }.force`, "[0, 2]\n"}, + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.take_while { |v| true }.force`, "[[0, 1], [2, 3]]\n"}, + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.drop_while { |v| false }.force`, "[[0, 1], [2, 3]]\n"}, + // select/reject/uniq see the gathered array + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +s = [] +o.to_enum.lazy.select { |v| s << v; true }.force +p s`, "[[0, 1], [2, 3]]\n"}, + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.uniq.force`, "[[0, 1], [2, 3]]\n"}, + // gathered multi-value survives a filter into a downstream map (sees first) + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.select { true }.map { |v| v }.force`, "[0, 2]\n"}, + // with_index gathers the value, no-block pairs it + {`o = Object.new +def o.each; yield 0, 1; yield 2, 3; end +p o.to_enum.lazy.with_index.force`, "[[[0, 1], 0], [[2, 3], 1]]\n"}, + + // --- flat_map flattening rules --- + {`p [1, 2].lazy.flat_map { |n| [n, -n] }.force`, "[1, -1, 2, -2]\n"}, + {`p [1, 2].lazy.flat_map { |n| n.to_s.each_char.lazy }.force`, "[\"1\", \"2\"]\n"}, + {`p [1, 2].lazy.flat_map { |n| n.to_s.each_char }.first(2).all? { |o| o.instance_of?(Enumerator) }`, "true\n"}, + + // --- with_index nil offset starts at 0 --- + {`p [10, 20].lazy.with_index(nil).force`, "[[10, 0], [20, 1]]\n"}, + {`p [10, 20].lazy.with_index(5).force`, "[[10, 5], [20, 6]]\n"}, + + // --- take quota / zero-count --- + {`$out = [] +o = Object.new +def o.each; $out << :before; yield 0; $out << :after; end +o.to_enum.lazy.take(1).force +p $out`, "[:before]\n"}, + {`$out = [] +o = Object.new +def o.each; $out << :before; yield 0; end +o.to_enum.lazy.take(0).force +p $out`, "[]\n"}, + {`p (1..Float::INFINITY).lazy.map { |x| x }.take(3).force`, "[1, 2, 3]\n"}, + // take followed by an early-stopping terminal unwinds through take + {`p (1..10).lazy.take(5).first(2)`, "[1, 2]\n"}, + + // --- eager/force still work --- + {`p [1, 2, 3].lazy.map { |x| x * 2 }.to_a`, "[2, 4, 6]\n"}, + // a lazy over a Chain enumerator pulls it fresh (forPull rewinds a chain) + {`p (1..2).each.chain([3, 4]).lazy.map { |x| x }.force`, "[1, 2, 3, 4]\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q\n got=%q\nwant=%q", c.src, got, c.want) + } + } + + errs := []struct{ src, want string }{ + // #feed twice without advancing raises TypeError + {`e = [1, 2].each; e.feed :a; e.feed :b`, "TypeError"}, + // Yielder#<< with more than one argument + {`Enumerator.new { |y| y.<<(1, 2) }.to_a`, "wrong number of arguments (given 2, expected 1)"}, + // each_with_index rejects extra arguments + {`[1].to_enum.each_with_index(:x)`, "ArgumentError"}, + // Enumerator::Lazy.new without a block + {`Enumerator::Lazy.new(Object.new)`, "ArgumentError"}, + // Enumerator::Lazy.new without a source + {`Enumerator::Lazy.new { |y| }`, "ArgumentError"}, + // lazy zip with a non-list argument + {`[1, 2].lazy.zip([], Object.new)`, "TypeError"}, + } + for _, c := range errs { + if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("src=%q got=%v want containing %q", c.src, err, c.want) + } + } +} diff --git a/internal/vm/lazy.go b/internal/vm/lazy.go index f076d2ee..5c05ad50 100644 --- a/internal/vm/lazy.go +++ b/internal/vm/lazy.go @@ -14,6 +14,16 @@ import ( type LazyEnum struct { recv object.Value ops []lazyOp + // sizeSpec, when sizeSpecSet, is the size given to Enumerator::Lazy.new(obj, + // size) { … }: it seeds #size (an Integer/nil verbatim, or a callable invoked on + // demand) in place of the source's own #size. Carried across #with so ops thread + // through it. When unset, #size starts from the source's #size (if any). + sizeSpec object.Value + sizeSpecSet bool + // ext caches the Enumerator that drives this pipeline for external iteration + // (#next/#peek/#rewind), pulling one element at a time through a fiber so the + // pipeline is never over-evaluated (and an infinite source stays usable). + ext *Enumerator } type lazyOp struct { @@ -38,13 +48,41 @@ func (l *LazyEnum) Truthy() bool { return true } // with returns a copy of l with one more transformation appended. func (l *LazyEnum) with(op lazyOp) *LazyEnum { ops := append(append([]lazyOp{}, l.ops...), op) - return &LazyEnum{recv: l.recv, ops: ops} + return &LazyEnum{recv: l.recv, ops: ops, sizeSpec: l.sizeSpec, sizeSpecSet: l.sizeSpecSet} } func (vm *VM) registerLazy() { vm.cLazy = newClass("Enumerator::Lazy", vm.cEnumerator) vm.cEnumerator.consts["Lazy"] = vm.cLazy + // Enumerator::Lazy.new(obj, size = nil) { |yielder, *values| … } builds a lazy + // enumerator whose block is run once per element of obj.each, receiving a yielder + // (its `<<`/`yield` feed the pipeline) followed by that element's yielded values. + // The optional size seeds #size (Integer/nil verbatim, or a callable on demand). + vm.cLazy.smethods["new"] = &Method{name: "new", owner: vm.cLazy, + native: func(_ *VM, _ object.Value, args []object.Value, blk *Proc) object.Value { + if blk == nil { + raise("ArgumentError", "tried to call lazy new without a block") + } + if len(args) == 0 { + raise("ArgumentError", "wrong number of arguments (given 0, expected 1+)") + } + src := args[0] + gen := &Proc{native: func(vm *VM, gargs []object.Value) object.Value { + y := gargs[0] + step := &Proc{native: func(vm *VM, vals []object.Value) object.Value { + vm.callBlock(blk, append([]object.Value{y}, vals...)) + return object.NilV + }} + return vm.send(src, "each", nil, step) + }} + le := &LazyEnum{recv: &Enumerator{block: gen}} + if len(args) > 1 { + le.sizeSpec, le.sizeSpecSet = args[1], true + } + return le + }} + makeLazy := func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { return &LazyEnum{recv: self} } @@ -70,13 +108,10 @@ func (vm *VM) registerLazy() { } } d("map", chain("map")) - d("collect", chain("map")) d("select", chain("select")) - d("filter", chain("select")) d("reject", chain("reject")) d("filter_map", chain("filter_map")) d("flat_map", chain("flat_map")) - d("collect_concat", chain("flat_map")) d("take_while", chain("take_while")) d("drop_while", chain("drop_while")) d("take", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { @@ -106,13 +141,24 @@ func (vm *VM) registerLazy() { }) // zip pairs each element with the corresponding elements of the other // sources (padding with nil once a source is exhausted). - d("zip", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + d("zip", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + // Each argument must be list-like (respond to #each); MRI validates this when + // #zip is called, before any element is pulled. + for _, a := range args { + if _, isArr := a.(*object.Array); !isArr && !vm.respondsToDynamic(a, "each") { + raise("TypeError", "wrong argument type %s (must respond to :each)", vm.classOf(a).name) + } + } return self.(*LazyEnum).with(lazyOp{kind: "zip", others: append([]object.Value{}, args...)}) }) // with_index(offset = 0): optional block maps (element, index); without a // block each element becomes the pair [element, index]. - d("with_index", func(_ *VM, self object.Value, args []object.Value, blk *Proc) object.Value { - return self.(*LazyEnum).with(lazyOp{kind: "with_index", n: int(intArgOr(args, 0)), blk: blk}) + d("with_index", func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value { + off := 0 + if len(args) > 0 && !object.IsNil(args[0]) { // a nil offset means "start at 0" + off = int(coerceInt(vm, args[0])) + } + return self.(*LazyEnum).with(lazyOp{kind: "with_index", n: off, blk: blk}) }) // chunk_while / slice_when split the stream into runs at each adjacent pair // (a, b) for which the block does not hold (chunk_while) / does hold @@ -197,24 +243,148 @@ func (vm *VM) registerLazy() { }) return l }) + // #size threads the source's size through the pipeline: filtering/reshaping ops + // (select, flat_map, grep, uniq, chunk, …) make it unknown (nil); map/with_index/ + // zip keep it; take/drop bound it — see lazyOpSize. + d("size", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return vm.lazySize(self.(*LazyEnum)) + }) + // External iteration over the lazy pipeline (#next/#peek/#rewind/…): driven by a + // cached Enumerator so each pull advances the pipeline one element at a time, + // never over-evaluating and never materialising an infinite source. + lazyExt := func(self object.Value) *Enumerator { + l := self.(*LazyEnum) + if l.ext == nil { + l.ext = &Enumerator{recv: l, meth: "each"} + } + return l.ext + } + d("next", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return enumPack(vm.enumNextRaw(lazyExt(self)).Elems) + }) + d("next_values", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return vm.enumNextRaw(lazyExt(self)) + }) + d("peek", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return enumPack(vm.enumPeekRaw(lazyExt(self)).Elems) + }) + d("peek_values", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return vm.enumPeekRaw(lazyExt(self)) + }) + d("rewind", func(vm *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + e := lazyExt(self) + e.extFiber, e.peeked, e.ended = nil, false, false + return self + }) + // enum_for / to_enum on a Lazy return a fresh Lazy that drives self.meth(*rest) + // lazily (through a generator Enumerator pulled one element at a time). A block + // supplies #size (returned verbatim, possibly nil) as elsewhere. + lazyToEnum := func(vm *VM, self object.Value, args []object.Value, blk *Proc) object.Value { + l := self.(*LazyEnum) + meth, rest := "each", []object.Value(nil) + if len(args) > 0 { + meth, rest = args[0].ToS(), args[1:] + } + gen := &Proc{native: func(vm *VM, gargs []object.Value) object.Value { + y := gargs[0] + step := &Proc{native: func(vm *VM, a []object.Value) object.Value { + vm.send(y, "yield", a, nil) + return object.NilV + }} + return vm.send(l, meth, rest, step) + }} + return &LazyEnum{recv: &Enumerator{block: gen, sizeBlock: blk}} + } + d("to_enum", lazyToEnum) + // Alias the shared Method objects so instance_method identities match MRI + // (Enumerator::Lazy#collect == #map, #filter == #select, …) and the Lazy method + // whitelist (instance_methods(false)) carries every expected name. + alias := func(name, of string) { vm.cLazy.methods[name] = vm.cLazy.methods[of] } + alias("collect", "map") + alias("filter", "select") + alias("find_all", "select") + alias("collect_concat", "flat_map") + alias("enum_for", "to_enum") +} + +// lazySize threads the source's #size through the op chain. A filtering or +// reshaping op makes the size unknown (nil); once unknown it stays unknown. +func (vm *VM) lazySize(l *LazyEnum) object.Value { + var sz object.Value + switch { + case l.sizeSpecSet: + sz = vm.enumResolveSize(l.sizeSpec) + case vm.respondsToDynamic(l.recv, "size"): + sz = vm.send(l.recv, "size", nil, nil) + default: + return object.NilV + } + // Only an Integer or positive-infinite Float is a size the op chain can reason + // about; anything else (nil, a finite Float, a non-numeric) is unknown. + if _, ok := sz.(object.Integer); !ok && !isInfFloat(sz) { + return object.NilV + } + for _, op := range l.ops { + sz = lazyOpSize(op, sz) + if object.IsNil(sz) { + return object.NilV + } + } + return sz +} + +// lazyOpSize maps a known incoming size (always an Integer or positive-infinite +// Float, per lazySize) through one op: map/with_index/zip keep it, take bounds it +// to its count (min, or the count itself for an infinite source), drop subtracts +// its count (flooring at 0, infinity staying infinite), and every +// filtering/reshaping op makes it unknown (nil). +func lazyOpSize(op lazyOp, sz object.Value) object.Value { + switch op.kind { + case "map", "with_index", "zip": + return sz + case "take": + n := int64(op.n) + if isInfFloat(sz) { + return object.IntValue(n) + } + if int64(sz.(object.Integer)) < n { + return sz + } + return object.IntValue(n) + case "drop": + n := int64(op.n) + if isInfFloat(sz) { + return sz + } + d := int64(sz.(object.Integer)) - n + if d < 0 { + d = 0 + } + return object.IntValue(d) + default: + return object.NilV + } } // lazySource returns a restartable pull function over le.recv: successive calls // yield the next source element until it returns ok=false. Integer ranges // (including endless and Float::INFINITY-bounded) are walked by counter; arrays // by index; an Enumerator is pulled one element at a time; any other Enumerable -// is materialised once (so it must be finite). -func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) { +// is materialised once (so it must be finite). The second result is the raw yield +// arguments when the source yielded several values at once (rb_yield_values2 +// semantics: a single-argument block then sees only the first) — nil for an +// ordinary single-value element. +func (vm *VM) lazySource(recv object.Value) func() (object.Value, []object.Value, bool) { switch r := recv.(type) { case *object.Array: i := 0 - return func() (object.Value, bool) { + return func() (object.Value, []object.Value, bool) { if i < len(r.Elems) { v := r.Elems[i] i++ - return v, true + return v, nil, true } - return object.NilVal(), false + return object.NilVal(), nil, false } case *object.Range: lo, ok := r.Lo.(object.Integer) @@ -239,13 +409,13 @@ func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) { if r.Exclusive && !unbounded { hi-- } - return func() (object.Value, bool) { + return func() (object.Value, []object.Value, bool) { if !unbounded && i > hi { - return object.NilVal(), false + return object.NilVal(), nil, false } v := object.IntValue(i) i++ - return v, true + return v, nil, true } case *Enumerator: // Pulled, not materialised. An Enumerator.new { |y| loop { … } } has no @@ -262,23 +432,29 @@ func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) { // abandoned #next does. That is the existing behaviour of external // iteration here, not something this adds. pull := r.forPull() - return func() (object.Value, bool) { + return func() (object.Value, []object.Value, bool) { w, ok := vm.enumPull(pull) if !ok { - return object.NilVal(), false + return object.NilVal(), nil, false + } + // A yield of several values (or none) is carried as raw args so a + // single-argument downstream block sees only the first; a lone value is + // an ordinary single-value element. + if len(w.Elems) == 1 { + return w.Elems[0], nil, true } - return enumPack(w.Elems), true + return enumPack(w.Elems), w.Elems, true } default: buf := vm.collectEach(recv) i := 0 - return func() (object.Value, bool) { + return func() (object.Value, []object.Value, bool) { if i < len(buf) { v := buf[i] i++ - return v, true + return v, nil, true } - return object.NilVal(), false + return object.NilVal(), nil, false } } } @@ -327,12 +503,12 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { src := vm.lazySource(le.recv) n := len(le.ops) // Per-op mutable run state. - rem := make([]int, n) // take / drop remaining - dropping := make([]bool, n) // drop_while latch - idx := make([]int, n) // with_index counter - seen := make([][]object.Value, n) // uniq keys - zpull := make([][]func() (object.Value, bool), n) // zip other sources - cst := make([]*chunkState, n) // grouping-op accumulators + rem := make([]int, n) // take / drop remaining + dropping := make([]bool, n) // drop_while latch + idx := make([]int, n) // with_index counter + seen := make([][]object.Value, n) // uniq keys + zpull := make([][]func() (object.Value, []object.Value, bool), n) // zip other sources + cst := make([]*chunkState, n) // grouping-op accumulators for i, op := range le.ops { switch op.kind { case "take", "drop": @@ -342,7 +518,7 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { case "with_index": idx[i] = op.n // starting offset case "zip": - ps := make([]func() (object.Value, bool), len(op.others)) + ps := make([]func() (object.Value, []object.Value, bool), len(op.others)) for j, o := range op.others { ps[j] = vm.lazySource(o) } @@ -352,9 +528,21 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } } stop := false - // feed threads v through ops[i:]; it returns false to abort the whole pull - // (want satisfied, or a take/take_while boundary reached). - var feed func(i int, v object.Value) bool + // feed threads one element through ops[i:]; it returns false to abort the whole + // pull (want satisfied, or a take/take_while boundary reached). v is the + // element's gathered value; multi, when non-nil, is the raw arguments a + // multi-value source yield produced (v == enumPack(multi)), so a block sees them + // by ordinary arity — a single-parameter block takes only the first. Values a + // filter passes through keep their multi; a value an op computes is single (nil). + var feed func(i int, v object.Value, multi []object.Value) bool + // blockArgs is the argument list to call an op's block with: the raw multi-value + // arguments when present, else the lone gathered value. + blockArgs := func(v object.Value, multi []object.Value) []object.Value { + if multi != nil { + return multi + } + return []object.Value{v} + } // emitBuf flushes a grouping op's buffered run downstream as one Array and // clears it; emitRun does the same for chunk's [key, [elems]] pair. emitBuf := func(i int, st *chunkState) bool { @@ -363,7 +551,7 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } grp := object.NewArrayFromSlice(st.buf) st.buf = nil - return feed(i+1, grp) + return feed(i+1, grp, nil) } emitRun := func(i int, st *chunkState) bool { if !st.open { @@ -372,9 +560,9 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { grp := object.NewArrayFromSlice([]object.Value{st.key, object.NewArrayFromSlice(st.buf)}) st.open = false st.buf = nil - return feed(i+1, grp) + return feed(i+1, grp, nil) } - feed = func(i int, v object.Value) bool { + feed = func(i int, v object.Value, multi []object.Value) bool { if i == n { if !sink(v) { stop = true @@ -385,52 +573,60 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { op := le.ops[i] switch op.kind { case "map": - return feed(i+1, vm.callBlock(op.blk, []object.Value{v})) + return feed(i+1, vm.callBlock(op.blk, blockArgs(v, multi)), nil) case "select": + // select/reject (like Enumerable#select) hand the block the *gathered* + // value of a multi-value yield, not the splatted arguments — unlike + // map/flat_map above — while still forwarding the multi downstream. if vm.callBlock(op.blk, []object.Value{v}).Truthy() { - return feed(i+1, v) + return feed(i+1, v, multi) } return true case "reject": if !vm.callBlock(op.blk, []object.Value{v}).Truthy() { - return feed(i+1, v) + return feed(i+1, v, multi) } return true case "filter_map": - w := vm.callBlock(op.blk, []object.Value{v}) + w := vm.callBlock(op.blk, blockArgs(v, multi)) if w.Truthy() { - return feed(i+1, w) + return feed(i+1, w, nil) } return true case "flat_map": - w := vm.callBlock(op.blk, []object.Value{v}) - if arr, ok := w.(*object.Array); ok { - for _, e := range arr.Elems { - if !feed(i+1, e) { + w := vm.callBlock(op.blk, blockArgs(v, multi)) + // MRI flattens an Array, or a value that responds to both #each and + // #force (an Enumerator::Lazy) — the latter forced to an Array first. A + // plain Enumerator (responds to #each but not #force) is passed through + // unflattened. + if arr, ok := lazyFlattenable(vm, w); ok { + for _, e := range arr { + if !feed(i+1, e, nil) { return false } } return true } - return feed(i+1, w) + return feed(i+1, w, nil) case "grep": if vm.send(op.pat, "===", []object.Value{v}, nil).Truthy() { - return feed(i+1, vm.lazyGrepValue(op.blk, v)) + return feed(i+1, vm.lazyGrepValue(op.blk, v), nil) } return true case "grep_v": if !vm.send(op.pat, "===", []object.Value{v}, nil).Truthy() { - return feed(i+1, vm.lazyGrepValue(op.blk, v)) + return feed(i+1, vm.lazyGrepValue(op.blk, v), nil) } return true case "compact": if _, isNil := v.(object.Nil); isNil { return true } - return feed(i+1, v) + return feed(i+1, v, multi) case "uniq": key := v if op.blk != nil { + // uniq's key block, like select's, sees the gathered value. key = vm.callBlock(op.blk, []object.Value{v}) } for _, k := range seen[i] { @@ -439,7 +635,7 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } } seen[i] = append(seen[i], key) - return feed(i+1, v) + return feed(i+1, v, multi) case "with_index": j := idx[i] idx[i]++ @@ -448,20 +644,20 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { // With a block, MRI evaluates it for its side effects and passes // the original item downstream (the block's result is ignored). vm.callBlock(op.blk, []object.Value{v, jv}) - return feed(i+1, v) + return feed(i+1, v, multi) } - return feed(i+1, object.NewArrayFromSlice([]object.Value{v, jv})) + return feed(i+1, object.NewArrayFromSlice([]object.Value{v, jv}), nil) case "zip": row := make([]object.Value, len(zpull[i])+1) row[0] = v for j, pf := range zpull[i] { - if e, ok := pf(); ok { + if e, _, ok := pf(); ok { row[j+1] = e } else { row[j+1] = object.NilV } } - return feed(i+1, object.NewArrayFromSlice(row)) + return feed(i+1, object.NewArrayFromSlice(row), nil) case "chunk_while": st := cst[i] if !st.hasPrev { @@ -515,7 +711,7 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { return false } pair := object.NewArrayFromSlice([]object.Value{k, object.NewArrayFromSlice([]object.Value{v})}) - return feed(i+1, pair) + return feed(i+1, pair, nil) } if st.open && valueEql(st.key, k) { st.buf = append(st.buf, v) @@ -561,26 +757,35 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } return true case "take_while": - if !vm.callBlock(op.blk, []object.Value{v}).Truthy() { + if !vm.callBlock(op.blk, blockArgs(v, multi)).Truthy() { stop = true return false } - return feed(i+1, v) + return feed(i+1, v, multi) case "drop_while": if dropping[i] { - if vm.callBlock(op.blk, []object.Value{v}).Truthy() { + if vm.callBlock(op.blk, blockArgs(v, multi)).Truthy() { return true } dropping[i] = false } - return feed(i+1, v) + return feed(i+1, v, multi) case "take": - if rem[i] <= 0 { + // A saturated take never reaches feed: the pre-loop takeSaturated check + // (and this op stopping the pull the moment its quota is met, below) keep + // rem[i] > 0 on entry, so take(0) drives the source zero times. + rem[i]-- + if !feed(i+1, v, multi) { + return false + } + // Stop pulling once the quota is met, rather than waiting to reject the + // next element — so take(n).force drives the source exactly n times (and + // never triggers a source that raises on its n+1-th step). + if rem[i] == 0 { stop = true return false } - rem[i]-- - return feed(i+1, v) + return true case "drop": if rem[i] > 0 { rem[i]-- @@ -588,7 +793,7 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } // Past the drop count: fall through to pass v downstream. } - return feed(i+1, v) + return feed(i+1, v, multi) } // finish flushes each grouping op's pending run once the source is exhausted, // threading it through the ops downstream (which may themselves group). @@ -609,12 +814,25 @@ func (vm *VM) lazyRun(le *LazyEnum, sink func(object.Value) bool) { } return finish(i + 1) } + // takeSaturated reports whether a take op has already met its quota, so the + // source need not be driven at all — take(0) then drives it zero times. + takeSaturated := func() bool { + for i, op := range le.ops { + if op.kind == "take" && rem[i] <= 0 { + return true + } + } + return false + } for !stop { - v, ok := src() + if takeSaturated() { + break + } + v, multi, ok := src() if !ok { break } - if !feed(0, v) { + if !feed(0, v, multi) { break } } @@ -646,6 +864,23 @@ func chunkKeyKind(k object.Value) (drop, alone, reserved bool) { return false, false, false } +// lazyFlattenable reports whether flat_map should flatten w, returning its +// elements when so: an Array flattens to its own elements; a value responding to +// both #each and #force (an Enumerator::Lazy) is forced to an Array and +// flattened. Anything else (including a plain Enumerator, which has #each but not +// #force) is not flattened. +func lazyFlattenable(vm *VM, w object.Value) ([]object.Value, bool) { + if arr, ok := w.(*object.Array); ok { + return arr.Elems, true + } + if vm.respondsToDynamic(w, "force") && vm.respondsToDynamic(w, "each") { + if arr, ok := vm.send(w, "force", nil, nil).(*object.Array); ok { + return arr.Elems, true + } + } + return nil, false +} + // lazyGrepValue returns the value grep/grep_v should emit for a match: the // element itself, or the block's mapping of it when a block was given. func (vm *VM) lazyGrepValue(blk *Proc, v object.Value) object.Value {