Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions internal/vm/enumerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,19 @@ type Enumerator struct {
finish object.Value
}

// forPull returns a copy of e carrying its definition and none of its
// external-iteration state, so it can be pulled from the start without moving
// the cursor #next and #peek share. See lazySource, which drives a lazy pipeline
// with it.
func (e *Enumerator) forPull() *Enumerator {
c := *e
c.extFiber, c.peekArgs, c.peeked, c.ended, c.finish = nil, nil, false, false, nil
if e.isChain {
c.entered = make([]bool, len(e.chainParts))
}
return &c
}

// yielder is the object passed to an Enumerator.new generator block; `y << v`
// and `y.yield(v)` feed values into the enumeration.
type yielder struct{ emit func(args []object.Value) }
Expand Down
25 changes: 24 additions & 1 deletion internal/vm/lazy.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,8 @@ func (vm *VM) registerLazy() {
// lazySource returns a restartable pull function over le.recv: successive calls
// yield the next source element until it returns ok=false. Integer ranges
// (including endless and Float::INFINITY-bounded) are walked by counter; arrays
// by index; any other Enumerable is materialised once (so it must be finite).
// by index; an Enumerator is pulled one element at a time; any other Enumerable
// is materialised once (so it must be finite).
func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) {
switch r := recv.(type) {
case *object.Array:
Expand Down Expand Up @@ -241,6 +242,28 @@ func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) {
i++
return v, true
}
case *Enumerator:
// Pulled, not materialised. An Enumerator.new { |y| loop { … } } has no
// end, and reading it whole allocated 8 GB before a CI runner died under
// it — which is the thing lazy exists to avoid, defeated at its source.
//
// The machinery is the one behind Enumerator#next: a fiber that runs the
// source until it yields and then suspends. Pulling runs on a copy of the
// enumerator, because each terminal operation starts from the beginning —
// e.lazy.first(3) twice gives the same three values, and would not if
// this advanced the caller's own cursor.
//
// A pipeline that stops early leaves that fiber suspended, exactly as an
// abandoned #next does. That is the existing behaviour of external
// iteration here, not something this adds.
pull := r.forPull()
return func() (object.Value, bool) {
w, ok := vm.enumPull(pull)
if !ok {
return object.NilVal(), false
}
return enumPack(w.Elems), true
}
default:
buf := vm.collectEach(recv)
i := 0
Expand Down
51 changes: 51 additions & 0 deletions internal/vm/lazy_generator_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright (c) the go-embedded-ruby/ruby authors
//
// SPDX-License-Identifier: BSD-3-Clause

package vm

import "testing"

// TestLazyOverAGeneratorIsLazy covers Enumerator::Lazy driven by an Enumerator
// rather than by an Array or a Range.
//
// lazySource materialised anything it did not recognise, and an
// Enumerator.new { |y| loop { … } } has no end: s.lazy.first(3) allocated 8 GB
// and killed a CI runner, which is lazy being defeated at its own source. It is
// pulled now, through the fiber behind Enumerator#next.
//
// Every expectation here was run against ruby 3 rather than reasoned about.
func TestLazyOverAGeneratorIsLazy(t *testing.T) {
cases := []struct{ src, want string }{
// The case that used to take the machine, in the three shapes the
// ruby/spec lazy files use.
{`s = Enumerator.new { |y| loop { y << 1 } }; p s.lazy.first(3)`, `[1, 1, 1]`},
{`s = Enumerator.new { |y| loop { y << 1 } }; p s.lazy.take(3).force`, `[1, 1, 1]`},
{`s = Enumerator.new { |y| loop { y << [1, 2] } }
p s.lazy.take(3).flat_map { |x| x }.force`, `[1, 2, 1, 2, 1, 2]`},
// A terminal operation starts from the beginning every time.
{`s = Enumerator.new { |y| loop { y << 1 } }
a = s.lazy.first(3); b = s.lazy.first(3); p [a, b]`, `[[1, 1, 1], [1, 1, 1]]`},
// And it leaves the enumerator's own #next cursor where it was, which is
// the reason the pull runs on a copy.
{`t = Enumerator.new { |y| y << 1; y << 2; y << 3 }
p t.next
p t.lazy.first(2)
p t.next`, "1\n[1, 2]\n2"},
// A finite generator still ends, and the ops still compose.
{`t = Enumerator.new { |y| y << 1; y << 2; y << 3 }
p t.lazy.map { |x| x * 10 }.to_a`, `[10, 20, 30]`},
{`t = Enumerator.new { |y| y << 1; y << 2; y << 3 }
p t.lazy.select { |x| x.odd? }.force`, `[1, 3]`},
// An Enumerator over something ordinary is pulled the same way.
{`p [1, 2, 3].each.lazy.map { |x| x + 1 }.force`, `[2, 3, 4]`},
// The sources that already worked still do.
{`p (0..Float::INFINITY).lazy.map { |n| n * 2 }.first(3)`, `[0, 2, 4]`},
{`p [1, 2, 3].lazy.map { |x| x }.force`, `[1, 2, 3]`},
}
for _, c := range cases {
if got := eval(t, c.src); got != c.want+"\n" {
t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n")
}
}
}