From 282aed6ccc539ec0f2e89ab4d1ef9fdae1aa3f4e Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:50:14 -0500 Subject: [PATCH 1/2] Reproduce backfiller starvation above shared capacity --- .../backfill_capacity_progress_test.go | 70 +++++++++++++++++++ docs/research/scheduler-backfill-capacity.md | 34 +++++++++ docs/research/scheduler-backfill-capacity.svg | 1 + 3 files changed, 105 insertions(+) create mode 100644 chasm/lib/scheduler/backfill_capacity_progress_test.go create mode 100644 docs/research/scheduler-backfill-capacity.md create mode 100644 docs/research/scheduler-backfill-capacity.svg diff --git a/chasm/lib/scheduler/backfill_capacity_progress_test.go b/chasm/lib/scheduler/backfill_capacity_progress_test.go new file mode 100644 index 00000000000..a2f919b9cff --- /dev/null +++ b/chasm/lib/scheduler/backfill_capacity_progress_test.go @@ -0,0 +1,70 @@ +package scheduler_test + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/common/clock" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestBackfillCapacityNativeControl(t *testing.T) { + testBackfillCapacityProgress(t, 450) +} + +func TestBackfillCapacityCounterexample(t *testing.T) { + if os.Getenv("TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES") != "1" { + t.Skip("set TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1") + } + for _, n := range []int{451, 1000} { + t.Run(fmt.Sprint(n), func(t *testing.T) { testBackfillCapacityProgress(t, n) }) + } +} + +func testBackfillCapacityProgress(t *testing.T, count int) { + t.Helper() + now := time.Date(2026, 9, 4, 12, 0, 0, 0, time.UTC) + ts := clock.NewEventTimeSource().Update(now) + spec := defaultSchedule() + spec.State.Paused = true + e := newSchedulerTestEngine(t, spec, withEngineTimeSource(ts)) + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + for range count { + s.NewRangeBackfiller(ctx, &schedulepb.BackfillRequest{ + StartTime: timestamppb.New(now), EndTime: timestamppb.New(now), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }) + } + return nil + })) + seen := make(map[string]string) + remaining := count + for round := 0; round < 50 && remaining > 0; round++ { + buffered := 0 + require.NoError(t, e.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + i := s.Invoker.Get(ctx) + buffered = len(i.BufferedStarts) + require.LessOrEqual(t, buffered, 460, "shared half-buffer minus generator reserve plus retained-history allowance") + for _, start := range i.BufferedStarts { + require.NotContains(t, seen, start.RequestId) + seen[start.RequestId] = start.WorkflowId + } + i.BufferedStarts = nil + remaining = len(s.Backfillers) + return nil + })) + require.Positive(t, buffered, "seed=capacity-%d: empty buffer and %d ranges must make progress", count, remaining) + ts.Update(ts.Now().Add(time.Hour)) + _, err := e.engine.FirePureTasks(e.rootRef, ts.Now()) + require.NoError(t, err) + } + require.Zero(t, remaining) + require.Len(t, seen, count) +} diff --git a/docs/research/scheduler-backfill-capacity.md b/docs/research/scheduler-backfill-capacity.md new file mode 100644 index 00000000000..96f70750c68 --- /dev/null +++ b/docs/research/scheduler-backfill-capacity.md @@ -0,0 +1,34 @@ +# Backfiller capacity truncation prevents progress + +Base: ad7b2298d. Seeds: `capacity-450`, `capacity-451`, `capacity-1000`. +Invariant: finite backfills with positive global capacity must eventually enqueue their actions without exceeding the shared budget. + +The component-engine test installs concurrent one-instant ranges on a paused minute schedule. All ranges share their boundary and ALLOW_ALL policy. It runs creation's immediate tasks and repeatedly drains admitted starts, advancing the logical clock to fire persisted continuations. The 450-range native control completes; 451 and 1000 ranges admit zero starts on their first pass despite an empty buffer. The sequence is deterministic; random backfiller UUIDs do not affect the invariant. Distinct request identities are checked on every drain. + +`allowedBufferedStarts` counts all range backfillers. At defaults, `backfillerBufferCapacity` computes `450 / count` for an empty buffer. At 451 the quotient becomes zero. Every task takes the capacity-stalled path, preserving its range and scheduling a backoff. Therefore the divisor never decreases and every subsequent attempt encounters the same state. + +Impact: V1 can carry up to 1000 ongoing backfills into CHASM even though native patch admission normally limits concurrency to 100. Migrated ranges above the arithmetic threshold remain stuck indefinitely. A configurable smaller buffer can also expose this below the default native concurrency limit. + +```mermaid +sequenceDiagram + participant Task as Backfiller task + participant Invoker + loop all 451 ranges, indefinitely + Task->>Invoker: read shared free capacity = 450 + Task->>Task: 450 / 451 = 0 + Task->>Task: retain range and reschedule + end +``` + +![Scrubbed failure](scheduler-backfill-capacity.svg) + +Control: `go test -tags test_dep ./chasm/lib/scheduler -run '^TestBackfillCapacityNativeControl$' -count=1`. +Counterexamples: `TEMPORAL_RUN_MIGRATION_COUNTEREXAMPLES=1 go test -tags test_dep ./chasm/lib/scheduler -run '^TestBackfillCapacityCounterexample$' -count=1`. + +The fix admits `min(available, max(1, available/count))`. Pure task mutations serialize on the scheduler tree, so every subsequent task recomputes capacity after the preceding enqueue. Zero global capacity still admits zero. The existing retained-history allowance is preserved, so the test's raw buffer bound is 460, equivalent to 450 pending slots after that allowance. This changes no protobuf or V1 workflow code and adds no new replay branch. + +Upstream audit: all open public PR titles/bodies fetched on 2026-09-04, with scheduler/backfill/migration candidates inspected. No matching production capacity fix found. Imported invoker activation is separately covered by #11557, and the fresh reverse-boundary defect by #11878. + +Ordering is separate from admission: native CHASM range task order is unspecified and reverse conversion iterates a map. Sorting ranges alone would not reproduce a guaranteed native order. Equal boundaries with distinct policies can therefore have order-dependent overlap outcomes; this report does not claim deterministic action equivalence across migration for that hypothesis. Capacity admission neither changes range cursors nor introduces a map-order guarantee. + +Crash/failover reasoning: enqueue, cursor advancement, and continuation task are one existing CHASM transaction. The change introduces no new external side effect or unreplicated state. These tests exercise committed component transactions, not actual History crashes or namespace failover. Counting all backfillers still costs O(N) per task; this fix addresses liveness, not that existing aggregate O(N²) scan cost. diff --git a/docs/research/scheduler-backfill-capacity.svg b/docs/research/scheduler-backfill-capacity.svg new file mode 100644 index 00000000000..09f9942b530 --- /dev/null +++ b/docs/research/scheduler-backfill-capacity.svg @@ -0,0 +1 @@ +FAIL: TestBackfillCapacityCounterexample/451seed=capacity-451: empty buffer and 451 ranges must make progressFAIL: TestBackfillCapacityCounterexample/1000seed=capacity-1000: empty buffer and 1000 ranges must make progressError: "0" is not positive From 13bf411727e0cdd46a96def86b2e66c97d96d0f0 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:52:30 -0500 Subject: [PATCH 2/2] Allow final backfiller cleanup without another buffered batch --- chasm/lib/scheduler/backfill_capacity_progress_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chasm/lib/scheduler/backfill_capacity_progress_test.go b/chasm/lib/scheduler/backfill_capacity_progress_test.go index a2f919b9cff..5af7124fe60 100644 --- a/chasm/lib/scheduler/backfill_capacity_progress_test.go +++ b/chasm/lib/scheduler/backfill_capacity_progress_test.go @@ -60,6 +60,9 @@ func testBackfillCapacityProgress(t *testing.T, count int) { remaining = len(s.Backfillers) return nil })) + if remaining == 0 { + break + } require.Positive(t, buffered, "seed=capacity-%d: empty buffer and %d ranges must make progress", count, remaining) ts.Update(ts.Now().Add(time.Hour)) _, err := e.engine.FirePureTasks(e.rootRef, ts.Now())