Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions src/main/java/com/knowledgepixels/nanodash/ApiCache.java
Original file line number Diff line number Diff line change
Expand Up @@ -271,12 +271,36 @@ static long backoffAfter(int consecutiveFailures) {
return FAILURE_BACKOFF_MS[i];
}

private static void recordFailure(String cacheId) {
/**
* How long a query is left alone after a failed attempt: the step of the ladder for the
* number of consecutive failures (see {@link #backoffAfter(int)}), or longer if the query
* servers refused the query and said when they would take it again. That time is theirs to
* name — a server refuses a query it is still evaluating, or has too many of — but never
* for longer than the ladder's last step, so that a wrong or far-off Retry-After cannot
* park a query for good.
*
* @param consecutiveFailures the number of failed attempts in a row, at least 1
* @param failure what the attempt failed with
* @param now the current time in milliseconds
* @return the delay in milliseconds before the next attempt
*/
static long backoffAfter(int consecutiveFailures, Throwable failure, long now) {
long backoff = backoffAfter(consecutiveFailures);
if (failure instanceof QueryRefusedException refused && refused.getRetryAfter() != null) {
long asked = refused.getRetryAfter().toEpochMilli() - now;
long longest = FAILURE_BACKOFF_MS[FAILURE_BACKOFF_MS.length - 1];
backoff = Math.min(Math.max(backoff, asked), longest);
}
return backoff;
}

private static void recordFailure(String cacheId, Throwable failure) {
long now = System.currentTimeMillis();
FailureState state = failures.asMap().merge(cacheId, new FailureState(1, now + backoffAfter(1)),
(previous, first) -> new FailureState(previous.count() + 1, now + backoffAfter(previous.count() + 1)));
FailureState state = failures.asMap().merge(cacheId, new FailureState(1, now + backoffAfter(1, failure, now)),
(previous, first) -> new FailureState(previous.count() + 1,
now + backoffAfter(previous.count() + 1, failure, now)));
logger.warn("Query failed {} time(s) in a row; not trying it again for {}s: {}",
state.count(), backoffAfter(state.count()) / 1000, cacheId);
state.count(), (state.retryAt() - now) / 1000, cacheId);
}

private static boolean isBackingOff(String cacheId) {
Expand Down Expand Up @@ -347,7 +371,7 @@ private static void runRefresh(String cacheId, long claim, boolean mayWait, Fetc
failures.invalidate(cacheId);
} catch (Exception ex) {
logger.error("Failed to update cache for {}: {}", cacheId, ex.getMessage());
recordFailure(cacheId);
recordFailure(cacheId, ex);
lastRefresh.put(cacheId, System.currentTimeMillis());
} finally {
refreshStart.remove(cacheId, claim);
Expand Down
10 changes: 8 additions & 2 deletions src/main/java/com/knowledgepixels/nanodash/QueryApiAccess.java
Original file line number Diff line number Diff line change
Expand Up @@ -368,12 +368,16 @@ private QueryApiAccess() {

/**
* Forces the retrieval of an API response for a given query name and parameters.
* Retries until a valid response is received.
* Retries for up to 30 seconds, until a valid response is received — unless the query
* servers refuse the query. A refusal is a server asking not to be sent this query for
* now, so asking again within the same attempt is exactly what it must not get; it goes
* to the caller instead, which knows from it when to try again (issue #889).
*
* @param queryRef The query reference
* @return The API response.
* @throws org.nanopub.extra.services.QueryRefusedException If the query servers refused the query.
*/
public static ApiResponse forcedGet(QueryRef queryRef) {
public static ApiResponse forcedGet(QueryRef queryRef) throws QueryRefusedException {
long deadline = System.currentTimeMillis() + 30_000;
long sleepMs = 1000;
while (System.currentTimeMillis() < deadline) {
Expand All @@ -382,6 +386,8 @@ public static ApiResponse forcedGet(QueryRef queryRef) {
if (resp != null) {
return resp;
}
} catch (QueryRefusedException ex) {
throw ex;
} catch (Exception ex) {
logger.error("Error while forcing API get for query {}", queryRef, ex);
}
Expand Down
37 changes: 37 additions & 0 deletions src/test/java/com/knowledgepixels/nanodash/ApiCacheTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@
import org.nanopub.extra.services.ApiResponse;
import org.nanopub.extra.services.FailedApiCallException;
import org.nanopub.extra.services.QueryRef;
import org.nanopub.extra.services.QueryRefusedException;

import com.google.common.cache.Cache;

import java.lang.reflect.Field;
import java.time.Instant;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
Expand Down Expand Up @@ -100,6 +102,10 @@ private Object publishRefresh(String npId, boolean awaitMaterialization, long re
}

/** Records the given number of consecutive failures, with the next attempt due at retryAt. */
private static QueryRefusedException refusedUntil(long epochMilli) {
return new QueryRefusedException("refused", Instant.ofEpochMilli(epochMilli));
}

private void putFailure(int count, long retryAt) throws Exception {
Class<?> c = Class.forName("com.knowledgepixels.nanodash.ApiCache$FailureState");
var ctor = c.getDeclaredConstructors()[0];
Expand Down Expand Up @@ -246,6 +252,37 @@ void backoffAfter_growsToFiveMinutes() {
assertEquals(300_000, ApiCache.backoffAfter(50));
}

@Test
@DisplayName("backoffAfter should wait as long as a refusing query server asks, but at least the ladder step and at most 5min (issue #889)")
void backoffAfter_honoursRetryAfter() {
long now = 1_000_000;
// Longer than the ladder step: the server's time wins.
assertEquals(45_000, ApiCache.backoffAfter(1, refusedUntil(now + 45_000), now));
// Shorter than the ladder step: the ladder still holds.
assertEquals(30_000, ApiCache.backoffAfter(2, refusedUntil(now + 5_000), now));
// Already past, or far off: bounded on both sides.
assertEquals(10_000, ApiCache.backoffAfter(1, refusedUntil(now - 60_000), now));
assertEquals(300_000, ApiCache.backoffAfter(1, refusedUntil(now + 3_600_000), now));
// A refusal that names no time, and any other failure, follow the ladder.
assertEquals(10_000, ApiCache.backoffAfter(1, new QueryRefusedException("refused", null), now));
assertEquals(60_000, ApiCache.backoffAfter(3, new FailedApiCallException(new Exception("timed out")), now));
}

@Test
@DisplayName("A refused query should not be tried again before the time the query server named (issue #889)")
void refusedQueryWaitsForRetryAfter() throws Exception {
try (MockedStatic<QueryApiAccess> queryApiAccess = mockStatic(QueryApiAccess.class)) {
long before = System.currentTimeMillis();
queryApiAccess.when(() -> QueryApiAccess.get(mockQueryRef)).thenThrow(refusedUntil(before + 45_000));

assertNull(ApiCache.retrieveResponseSync(mockQueryRef, false));

assertEquals(1, failureCount());
assertTrue(retryAt() >= before + 44_000, "the first failure alone would only wait 10s");
assertTrue(retryAt() <= System.currentTimeMillis() + 45_000);
}
}

@Test
@DisplayName("retrieveResponseSync should make no call while the query is backed off, forced or not, and keep the backoff (issue #889)")
void retrieveResponseSync_makesNoCallWhileBackingOff() throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ void getReturnsApiResponseForValidQueryId() throws FailedApiCallException, APINo
}

@Test
void forcedGetRetriesUntilApiResponseIsNotNull() {
void forcedGetRetriesUntilApiResponseIsNotNull() throws Exception {
ApiResponse expectedResponse = new ApiResponse();

try (MockedStatic<QueryAccess> mockQueryAccess = mockStatic(QueryAccess.class)) {
Expand All @@ -45,6 +45,21 @@ void forcedGetRetriesUntilApiResponseIsNotNull() {
}
}

@Test
void forcedGetDoesNotAskAgainWhenTheQueryIsRefused() {
QueryRefusedException refusal = new QueryRefusedException("refused", java.time.Instant.now().plusSeconds(60));

try (MockedStatic<QueryAccess> mockQueryAccess = mockStatic(QueryAccess.class)) {
mockQueryAccess.when(() -> QueryAccess.get(any(QueryRef.class))).thenThrow(refusal);

QueryRefusedException thrown = assertThrows(QueryRefusedException.class,
() -> QueryApiAccess.forcedGet(new QueryRef(queryId)));

assertSame(refusal, thrown);
mockQueryAccess.verify(() -> QueryAccess.get(any(QueryRef.class)), org.mockito.Mockito.times(1));
}
}

@Test
void getThrowsExceptionWhenNameIsNotFullQueryId() {
assertThrows(IllegalArgumentException.class, () -> QueryApiAccess.get(new QueryRef("short-name-only")));
Expand Down
Loading