From 16e8bca66fa95002c78f100650f1c0e9303f0739 Mon Sep 17 00:00:00 2001 From: Tobias Kuhn Date: Fri, 13 Nov 2026 15:01:25 +0100 Subject: [PATCH] fix(api-cache): wait as long as a refusing query server asks (#889) A query server that refuses a query (503/429 with Retry-After) names the time it will take the query again. nanopub-java 1.96.0 passes that on as QueryRefusedException: - ApiCache waits until that time before the next attempt, though never less than the backoff step for the failure count and never more than five minutes. - QueryApiAccess.forcedGet hands a refusal to the caller instead of asking again for 30 seconds, which sent a refusing server about 15 requests per attempt. Co-Authored-By: Claude Fable 5.2 --- .../knowledgepixels/nanodash/ApiCache.java | 34 ++++++++++++++--- .../nanodash/QueryApiAccess.java | 10 ++++- .../nanodash/ApiCacheTest.java | 37 +++++++++++++++++++ .../nanodash/QueryApiAccessTest.java | 17 ++++++++- 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/knowledgepixels/nanodash/ApiCache.java b/src/main/java/com/knowledgepixels/nanodash/ApiCache.java index 5c685fb1c..bdf2fa01e 100644 --- a/src/main/java/com/knowledgepixels/nanodash/ApiCache.java +++ b/src/main/java/com/knowledgepixels/nanodash/ApiCache.java @@ -271,12 +271,36 @@ static long backoffAfter(int consecutiveFailures) { return FAILURE_BACKOFF_MS[i]; } - private static void recordFailure(String cacheId) { + /** + * How long a query is left alone after a failed attempt: the step of the ladder for the + * number of consecutive failures (see {@link #backoffAfter(int)}), or longer if the query + * servers refused the query and said when they would take it again. That time is theirs to + * name — a server refuses a query it is still evaluating, or has too many of — but never + * for longer than the ladder's last step, so that a wrong or far-off Retry-After cannot + * park a query for good. + * + * @param consecutiveFailures the number of failed attempts in a row, at least 1 + * @param failure what the attempt failed with + * @param now the current time in milliseconds + * @return the delay in milliseconds before the next attempt + */ + static long backoffAfter(int consecutiveFailures, Throwable failure, long now) { + long backoff = backoffAfter(consecutiveFailures); + if (failure instanceof QueryRefusedException refused && refused.getRetryAfter() != null) { + long asked = refused.getRetryAfter().toEpochMilli() - now; + long longest = FAILURE_BACKOFF_MS[FAILURE_BACKOFF_MS.length - 1]; + backoff = Math.min(Math.max(backoff, asked), longest); + } + return backoff; + } + + private static void recordFailure(String cacheId, Throwable failure) { long now = System.currentTimeMillis(); - FailureState state = failures.asMap().merge(cacheId, new FailureState(1, now + backoffAfter(1)), - (previous, first) -> new FailureState(previous.count() + 1, now + backoffAfter(previous.count() + 1))); + FailureState state = failures.asMap().merge(cacheId, new FailureState(1, now + backoffAfter(1, failure, now)), + (previous, first) -> new FailureState(previous.count() + 1, + now + backoffAfter(previous.count() + 1, failure, now))); logger.warn("Query failed {} time(s) in a row; not trying it again for {}s: {}", - state.count(), backoffAfter(state.count()) / 1000, cacheId); + state.count(), (state.retryAt() - now) / 1000, cacheId); } private static boolean isBackingOff(String cacheId) { @@ -347,7 +371,7 @@ private static void runRefresh(String cacheId, long claim, boolean mayWait, Fetc failures.invalidate(cacheId); } catch (Exception ex) { logger.error("Failed to update cache for {}: {}", cacheId, ex.getMessage()); - recordFailure(cacheId); + recordFailure(cacheId, ex); lastRefresh.put(cacheId, System.currentTimeMillis()); } finally { refreshStart.remove(cacheId, claim); diff --git a/src/main/java/com/knowledgepixels/nanodash/QueryApiAccess.java b/src/main/java/com/knowledgepixels/nanodash/QueryApiAccess.java index 8fdefc193..39df33c39 100644 --- a/src/main/java/com/knowledgepixels/nanodash/QueryApiAccess.java +++ b/src/main/java/com/knowledgepixels/nanodash/QueryApiAccess.java @@ -368,12 +368,16 @@ private QueryApiAccess() { /** * Forces the retrieval of an API response for a given query name and parameters. - * Retries until a valid response is received. + * Retries for up to 30 seconds, until a valid response is received — unless the query + * servers refuse the query. A refusal is a server asking not to be sent this query for + * now, so asking again within the same attempt is exactly what it must not get; it goes + * to the caller instead, which knows from it when to try again (issue #889). * * @param queryRef The query reference * @return The API response. + * @throws org.nanopub.extra.services.QueryRefusedException If the query servers refused the query. */ - public static ApiResponse forcedGet(QueryRef queryRef) { + public static ApiResponse forcedGet(QueryRef queryRef) throws QueryRefusedException { long deadline = System.currentTimeMillis() + 30_000; long sleepMs = 1000; while (System.currentTimeMillis() < deadline) { @@ -382,6 +386,8 @@ public static ApiResponse forcedGet(QueryRef queryRef) { if (resp != null) { return resp; } + } catch (QueryRefusedException ex) { + throw ex; } catch (Exception ex) { logger.error("Error while forcing API get for query {}", queryRef, ex); } diff --git a/src/test/java/com/knowledgepixels/nanodash/ApiCacheTest.java b/src/test/java/com/knowledgepixels/nanodash/ApiCacheTest.java index 7a36d65b8..a7c80bb0d 100644 --- a/src/test/java/com/knowledgepixels/nanodash/ApiCacheTest.java +++ b/src/test/java/com/knowledgepixels/nanodash/ApiCacheTest.java @@ -12,10 +12,12 @@ import org.nanopub.extra.services.ApiResponse; import org.nanopub.extra.services.FailedApiCallException; import org.nanopub.extra.services.QueryRef; +import org.nanopub.extra.services.QueryRefusedException; import com.google.common.cache.Cache; import java.lang.reflect.Field; +import java.time.Instant; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentMap; @@ -100,6 +102,10 @@ private Object publishRefresh(String npId, boolean awaitMaterialization, long re } /** Records the given number of consecutive failures, with the next attempt due at retryAt. */ + private static QueryRefusedException refusedUntil(long epochMilli) { + return new QueryRefusedException("refused", Instant.ofEpochMilli(epochMilli)); + } + private void putFailure(int count, long retryAt) throws Exception { Class c = Class.forName("com.knowledgepixels.nanodash.ApiCache$FailureState"); var ctor = c.getDeclaredConstructors()[0]; @@ -246,6 +252,37 @@ void backoffAfter_growsToFiveMinutes() { assertEquals(300_000, ApiCache.backoffAfter(50)); } + @Test + @DisplayName("backoffAfter should wait as long as a refusing query server asks, but at least the ladder step and at most 5min (issue #889)") + void backoffAfter_honoursRetryAfter() { + long now = 1_000_000; + // Longer than the ladder step: the server's time wins. + assertEquals(45_000, ApiCache.backoffAfter(1, refusedUntil(now + 45_000), now)); + // Shorter than the ladder step: the ladder still holds. + assertEquals(30_000, ApiCache.backoffAfter(2, refusedUntil(now + 5_000), now)); + // Already past, or far off: bounded on both sides. + assertEquals(10_000, ApiCache.backoffAfter(1, refusedUntil(now - 60_000), now)); + assertEquals(300_000, ApiCache.backoffAfter(1, refusedUntil(now + 3_600_000), now)); + // A refusal that names no time, and any other failure, follow the ladder. + assertEquals(10_000, ApiCache.backoffAfter(1, new QueryRefusedException("refused", null), now)); + assertEquals(60_000, ApiCache.backoffAfter(3, new FailedApiCallException(new Exception("timed out")), now)); + } + + @Test + @DisplayName("A refused query should not be tried again before the time the query server named (issue #889)") + void refusedQueryWaitsForRetryAfter() throws Exception { + try (MockedStatic queryApiAccess = mockStatic(QueryApiAccess.class)) { + long before = System.currentTimeMillis(); + queryApiAccess.when(() -> QueryApiAccess.get(mockQueryRef)).thenThrow(refusedUntil(before + 45_000)); + + assertNull(ApiCache.retrieveResponseSync(mockQueryRef, false)); + + assertEquals(1, failureCount()); + assertTrue(retryAt() >= before + 44_000, "the first failure alone would only wait 10s"); + assertTrue(retryAt() <= System.currentTimeMillis() + 45_000); + } + } + @Test @DisplayName("retrieveResponseSync should make no call while the query is backed off, forced or not, and keep the backoff (issue #889)") void retrieveResponseSync_makesNoCallWhileBackingOff() throws Exception { diff --git a/src/test/java/com/knowledgepixels/nanodash/QueryApiAccessTest.java b/src/test/java/com/knowledgepixels/nanodash/QueryApiAccessTest.java index ddc93abc8..58c38aef2 100644 --- a/src/test/java/com/knowledgepixels/nanodash/QueryApiAccessTest.java +++ b/src/test/java/com/knowledgepixels/nanodash/QueryApiAccessTest.java @@ -30,7 +30,7 @@ void getReturnsApiResponseForValidQueryId() throws FailedApiCallException, APINo } @Test - void forcedGetRetriesUntilApiResponseIsNotNull() { + void forcedGetRetriesUntilApiResponseIsNotNull() throws Exception { ApiResponse expectedResponse = new ApiResponse(); try (MockedStatic mockQueryAccess = mockStatic(QueryAccess.class)) { @@ -45,6 +45,21 @@ void forcedGetRetriesUntilApiResponseIsNotNull() { } } + @Test + void forcedGetDoesNotAskAgainWhenTheQueryIsRefused() { + QueryRefusedException refusal = new QueryRefusedException("refused", java.time.Instant.now().plusSeconds(60)); + + try (MockedStatic mockQueryAccess = mockStatic(QueryAccess.class)) { + mockQueryAccess.when(() -> QueryAccess.get(any(QueryRef.class))).thenThrow(refusal); + + QueryRefusedException thrown = assertThrows(QueryRefusedException.class, + () -> QueryApiAccess.forcedGet(new QueryRef(queryId))); + + assertSame(refusal, thrown); + mockQueryAccess.verify(() -> QueryAccess.get(any(QueryRef.class)), org.mockito.Mockito.times(1)); + } + } + @Test void getThrowsExceptionWhenNameIsNotFullQueryId() { assertThrows(IllegalArgumentException.class, () -> QueryApiAccess.get(new QueryRef("short-name-only")));