From ce6072a3c5a059d5f1339416fd5825cc6776ff9b Mon Sep 17 00:00:00 2001 From: tannevaled Date: Fri, 21 Aug 2026 08:54:11 +0200 Subject: [PATCH] Enumerator::Lazy: pull an Enumerator source rather than read it whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lazySource walked a Range by counter and an Array by index, and materialised anything else — "so it must be finite", as its own comment said. An Enumerator.new { |y| loop { … } } is not finite, so lazy was defeated at its source: s.lazy.first(3) allocated 8 GB and killed a CI runner. It is the fourth and last of the reasons the rubyspec-ratchet lane was red. It is pulled now, through the fiber behind Enumerator#next — machinery that already existed and already handled an endless generator: s.next three times costs 79 MB. Only the lazy path was not using it. s.lazy.first(3) before 7.4 GB after 79 MB s.lazy.take(3).force before 8.0 GB after 79 MB s.lazy.take(3).flat_map{}.force before 12.7 GB after 80 MB core/enumerator/lazy/flat_map_spec.rb goes from 48.2 GB to 78 MB. The pull runs on a copy of the enumerator, carrying its definition and none of its external-iteration state. That is what makes a terminal operation start from the beginning every time, and leaves the caller's own #next cursor where it was — both checked against ruby 3, which agrees line for line, including next/lazy/next interleaved. --- internal/vm/enumerator.go | 13 ++++++ internal/vm/lazy.go | 25 +++++++++- internal/vm/lazy_generator_internal_test.go | 51 +++++++++++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 internal/vm/lazy_generator_internal_test.go diff --git a/internal/vm/enumerator.go b/internal/vm/enumerator.go index 62d15bcf..0d259241 100644 --- a/internal/vm/enumerator.go +++ b/internal/vm/enumerator.go @@ -55,6 +55,19 @@ type Enumerator struct { finish object.Value } +// forPull returns a copy of e carrying its definition and none of its +// external-iteration state, so it can be pulled from the start without moving +// the cursor #next and #peek share. See lazySource, which drives a lazy pipeline +// with it. +func (e *Enumerator) forPull() *Enumerator { + c := *e + c.extFiber, c.peekArgs, c.peeked, c.ended, c.finish = nil, nil, false, false, nil + if e.isChain { + c.entered = make([]bool, len(e.chainParts)) + } + return &c +} + // yielder is the object passed to an Enumerator.new generator block; `y << v` // and `y.yield(v)` feed values into the enumeration. type yielder struct{ emit func(args []object.Value) } diff --git a/internal/vm/lazy.go b/internal/vm/lazy.go index 9f28e8ef..e05a01ee 100644 --- a/internal/vm/lazy.go +++ b/internal/vm/lazy.go @@ -197,7 +197,8 @@ func (vm *VM) registerLazy() { // lazySource returns a restartable pull function over le.recv: successive calls // yield the next source element until it returns ok=false. Integer ranges // (including endless and Float::INFINITY-bounded) are walked by counter; arrays -// by index; any other Enumerable is materialised once (so it must be finite). +// by index; an Enumerator is pulled one element at a time; any other Enumerable +// is materialised once (so it must be finite). func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) { switch r := recv.(type) { case *object.Array: @@ -241,6 +242,28 @@ func (vm *VM) lazySource(recv object.Value) func() (object.Value, bool) { i++ return v, true } + case *Enumerator: + // Pulled, not materialised. An Enumerator.new { |y| loop { … } } has no + // end, and reading it whole allocated 8 GB before a CI runner died under + // it — which is the thing lazy exists to avoid, defeated at its source. + // + // The machinery is the one behind Enumerator#next: a fiber that runs the + // source until it yields and then suspends. Pulling runs on a copy of the + // enumerator, because each terminal operation starts from the beginning — + // e.lazy.first(3) twice gives the same three values, and would not if + // this advanced the caller's own cursor. + // + // A pipeline that stops early leaves that fiber suspended, exactly as an + // abandoned #next does. That is the existing behaviour of external + // iteration here, not something this adds. + pull := r.forPull() + return func() (object.Value, bool) { + w, ok := vm.enumPull(pull) + if !ok { + return object.NilVal(), false + } + return enumPack(w.Elems), true + } default: buf := vm.collectEach(recv) i := 0 diff --git a/internal/vm/lazy_generator_internal_test.go b/internal/vm/lazy_generator_internal_test.go new file mode 100644 index 00000000..d55fead9 --- /dev/null +++ b/internal/vm/lazy_generator_internal_test.go @@ -0,0 +1,51 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +package vm + +import "testing" + +// TestLazyOverAGeneratorIsLazy covers Enumerator::Lazy driven by an Enumerator +// rather than by an Array or a Range. +// +// lazySource materialised anything it did not recognise, and an +// Enumerator.new { |y| loop { … } } has no end: s.lazy.first(3) allocated 8 GB +// and killed a CI runner, which is lazy being defeated at its own source. It is +// pulled now, through the fiber behind Enumerator#next. +// +// Every expectation here was run against ruby 3 rather than reasoned about. +func TestLazyOverAGeneratorIsLazy(t *testing.T) { + cases := []struct{ src, want string }{ + // The case that used to take the machine, in the three shapes the + // ruby/spec lazy files use. + {`s = Enumerator.new { |y| loop { y << 1 } }; p s.lazy.first(3)`, `[1, 1, 1]`}, + {`s = Enumerator.new { |y| loop { y << 1 } }; p s.lazy.take(3).force`, `[1, 1, 1]`}, + {`s = Enumerator.new { |y| loop { y << [1, 2] } } +p s.lazy.take(3).flat_map { |x| x }.force`, `[1, 2, 1, 2, 1, 2]`}, + // A terminal operation starts from the beginning every time. + {`s = Enumerator.new { |y| loop { y << 1 } } +a = s.lazy.first(3); b = s.lazy.first(3); p [a, b]`, `[[1, 1, 1], [1, 1, 1]]`}, + // And it leaves the enumerator's own #next cursor where it was, which is + // the reason the pull runs on a copy. + {`t = Enumerator.new { |y| y << 1; y << 2; y << 3 } +p t.next +p t.lazy.first(2) +p t.next`, "1\n[1, 2]\n2"}, + // A finite generator still ends, and the ops still compose. + {`t = Enumerator.new { |y| y << 1; y << 2; y << 3 } +p t.lazy.map { |x| x * 10 }.to_a`, `[10, 20, 30]`}, + {`t = Enumerator.new { |y| y << 1; y << 2; y << 3 } +p t.lazy.select { |x| x.odd? }.force`, `[1, 3]`}, + // An Enumerator over something ordinary is pulled the same way. + {`p [1, 2, 3].each.lazy.map { |x| x + 1 }.force`, `[2, 3, 4]`}, + // The sources that already worked still do. + {`p (0..Float::INFINITY).lazy.map { |n| n * 2 }.first(3)`, `[0, 2, 4]`}, + {`p [1, 2, 3].lazy.map { |x| x }.force`, `[1, 2, 3]`}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want+"\n" { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want+"\n") + } + } +}