From 6ca66ca11f6a59816ebcfbb287a3b12b421a0ecf Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 8 Jul 2026 15:19:29 +1200 Subject: [PATCH 1/3] test: failing tests for superseding event not resetting retry counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A superseding event arriving during a retry cycle does not reset the retry counter. If the superseding event's re-execution also fails (e.g. because the informer cache is still stale), the retry budget from the original cycle is inherited and may already be exhausted, leaving the resource permanently un-reconciled. Two failing tests demonstrate this: 1. newEventShouldBeRetryableAfterPriorRetryExhaustion — unit-level proof that a new event after retry exhaustion inherits the exhausted counter. 2. supersedingEventConsumedDuringRetryShouldNotPermanentlyStallReconciliation — uses a real TimerEventSource and CountDownLatch to reproduce the end-to-end scenario: initial failure → timer retry (blocked by latch) → superseding event injected mid-retry → re-execution fails → retry budget exhausted → resource permanently stuck. Both tests are @Disabled pending a fix. Assisted-by: Claude claude-opus-4-6 Signed-off-by: Sam Barker --- .../processing/event/EventProcessorTest.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java index f7864f2f16..7ebba4f2c2 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java @@ -19,8 +19,12 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.internal.stubbing.answers.AnswersWithDelay; @@ -244,6 +248,111 @@ private void waitUntilProcessingFinished( .until(() -> !eventProcessor.isUnderProcessing(relatedCustomResourceID)); } + @Disabled( + "Superseding events do not reset the retry counter — new events inherit exhausted budget") + @Test + void newEventShouldBeRetryableAfterPriorRetryExhaustion() { + // Given + EventProcessor processor = + spy( + new EventProcessor( + controllerConfigurationNoMaxReconciliationInterval( + new GenericRetry().setMaxAttempts(1).setInitialInterval(2000), rateLimiterMock), + reconciliationDispatcherMock, + eventSourceManagerMock, + null)); + processor.start(); + when(processor.retryEventSource()).thenReturn(retryTimerEventSourceMock); + + TestCustomResource customResource = testCustomResource(); + ResourceID resourceID = ResourceID.fromResource(customResource); + when(controllerEventSourceMock.get(eq(resourceID))).thenReturn(Optional.of(customResource)); + + ExecutionScope priorFailure = new ExecutionScope(customResource, null, false, false); + processor.eventProcessingFinished( + priorFailure, PostExecutionControl.exceptionDuringExecution(new RuntimeException("test"))); + + when(reconciliationDispatcherMock.handleExecution(any())) + .thenReturn( + PostExecutionControl.exceptionDuringExecution( + new RuntimeException("informer cache stale"))); + + // When + processor.handleEvent(new ResourceEvent(ResourceAction.UPDATED, resourceID, customResource)); + + // Then + verify(reconciliationDispatcherMock, timeout(SEPARATE_EXECUTION_TIMEOUT).times(1)) + .handleExecution(any()); + waitUntilProcessingFinished(processor, resourceID); + + // The new event's failed execution should still be retryable. Without + // maxReconciliationInterval there is no fallback — if the retry budget from the prior + // cycle is inherited and already exhausted, no retry is scheduled and the resource + // is permanently stuck. + verify(retryTimerEventSourceMock, times(2)).scheduleOnce(eq(resourceID), anyLong()); + } + + @Disabled( + "Superseding events do not reset the retry counter — new events inherit exhausted budget") + @Test + void supersedingEventConsumedDuringRetryShouldNotPermanentlyStallReconciliation() + throws Exception { + // Given + TimerEventSource realTimerSource = new TimerEventSource<>(); + + EventProcessor processor = + spy( + new EventProcessor<>( + controllerConfigurationNoMaxReconciliationInterval( + new GenericRetry().setMaxAttempts(1).setInitialInterval(100), rateLimiterMock), + reconciliationDispatcherMock, + eventSourceManagerMock, + null)); + + when(processor.retryEventSource()).thenReturn(realTimerSource); + realTimerSource.setEventHandler(processor); + realTimerSource.start(); + processor.start(); + + try { + TestCustomResource customResource = testCustomResource(); + ResourceID resourceID = ResourceID.fromResource(customResource); + when(controllerEventSourceMock.get(eq(resourceID))).thenReturn(Optional.of(customResource)); + + CountDownLatch retryExecutionStarted = new CountDownLatch(1); + CountDownLatch proceedWithRetry = new CountDownLatch(1); + AtomicInteger executionCount = new AtomicInteger(0); + + when(reconciliationDispatcherMock.handleExecution(any())) + .thenAnswer( + invocation -> { + int count = executionCount.incrementAndGet(); + if (count == 2) { + retryExecutionStarted.countDown(); + proceedWithRetry.await(5, TimeUnit.SECONDS); + } + if (count <= 3) { + return PostExecutionControl.exceptionDuringExecution( + new RuntimeException("informer cache stale")); + } + return PostExecutionControl.defaultDispatch(); + }); + + processor.handleEvent(new ResourceEvent(ResourceAction.UPDATED, resourceID, customResource)); + retryExecutionStarted.await(5, TimeUnit.SECONDS); + + // When + processor.handleEvent(new ResourceEvent(ResourceAction.UPDATED, resourceID, customResource)); + + // Then + proceedWithRetry.countDown(); + verify(reconciliationDispatcherMock, timeout(5000).times(4)).handleExecution(any()); + } finally { + processor.stop(); + realTimerSource.stop(); + } + } + @Test void scheduleTimedEventIfInstructedByPostExecutionControl() { var testDelay = 10000L; @@ -900,6 +1009,18 @@ ControllerConfiguration controllerConfigTriggerAllEvent(Retry retry, RateLimiter return controllerConfiguration(retry, rateLimiter, new BaseConfigurationService(), true); } + ControllerConfiguration controllerConfigurationNoMaxReconciliationInterval( + Retry retry, RateLimiter rateLimiter) { + ControllerConfiguration res = mock(ControllerConfiguration.class); + when(res.getName()).thenReturn("Test"); + when(res.getRetry()).thenReturn(retry); + when(res.getRateLimiter()).thenReturn(rateLimiter); + when(res.maxReconciliationInterval()).thenReturn(Optional.empty()); + when(res.getConfigurationService()).thenReturn(new BaseConfigurationService()); + when(res.triggerReconcilerOnAllEvents()).thenReturn(false); + return res; + } + ControllerConfiguration controllerConfiguration( Retry retry, RateLimiter rateLimiter, From 0aa98a9318124b0367b482f1cee586528876736d Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 8 Jul 2026 16:11:45 +1200 Subject: [PATCH 2/3] chore: link @Disabled annotations to issue #3479 Assisted-by: Claude claude-opus-4-6 Signed-off-by: Sam Barker --- .../operator/processing/event/EventProcessorTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java index 7ebba4f2c2..e606a5cb44 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java @@ -248,8 +248,7 @@ private void waitUntilProcessingFinished( .until(() -> !eventProcessor.isUnderProcessing(relatedCustomResourceID)); } - @Disabled( - "Superseding events do not reset the retry counter — new events inherit exhausted budget") + @Disabled("https://github.com/operator-framework/java-operator-sdk/issues/3479") @Test void newEventShouldBeRetryableAfterPriorRetryExhaustion() { // Given @@ -292,8 +291,7 @@ void newEventShouldBeRetryableAfterPriorRetryExhaustion() { verify(retryTimerEventSourceMock, times(2)).scheduleOnce(eq(resourceID), anyLong()); } - @Disabled( - "Superseding events do not reset the retry counter — new events inherit exhausted budget") + @Disabled("https://github.com/operator-framework/java-operator-sdk/issues/3479") @Test void supersedingEventConsumedDuringRetryShouldNotPermanentlyStallReconciliation() throws Exception { From 2b4dfd90e7afd4e284c768b9145ab1afce74f431 Mon Sep 17 00:00:00 2001 From: Sam Barker Date: Wed, 8 Jul 2026 16:15:58 +1200 Subject: [PATCH 3/3] fix: assert CountDownLatch await results to fail fast on timeout Assisted-by: Claude claude-opus-4-6 Signed-off-by: Sam Barker --- .../operator/processing/event/EventProcessorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java index e606a5cb44..c786c64d4a 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorTest.java @@ -327,7 +327,7 @@ void supersedingEventConsumedDuringRetryShouldNotPermanentlyStallReconciliation( int count = executionCount.incrementAndGet(); if (count == 2) { retryExecutionStarted.countDown(); - proceedWithRetry.await(5, TimeUnit.SECONDS); + assertThat(proceedWithRetry.await(5, TimeUnit.SECONDS)).isTrue(); } if (count <= 3) { return PostExecutionControl.exceptionDuringExecution( @@ -337,7 +337,7 @@ void supersedingEventConsumedDuringRetryShouldNotPermanentlyStallReconciliation( }); processor.handleEvent(new ResourceEvent(ResourceAction.UPDATED, resourceID, customResource)); - retryExecutionStarted.await(5, TimeUnit.SECONDS); + assertThat(retryExecutionStarted.await(5, TimeUnit.SECONDS)).isTrue(); // When processor.handleEvent(new ResourceEvent(ResourceAction.UPDATED, resourceID, customResource));