From c3ca6c1e3456f3fe79440fa090ffbe8d73559576 Mon Sep 17 00:00:00 2001
From: Yaniv Kaul <yaniv.kaul@scylladb.com>
Date: Sun, 13 Sep 2026 00:20:32 +0300
Subject: [PATCH] tests: retry client-routes POST on transient 5xx after
 decommission

A node's REST API can briefly return HTTP 500 for /v2/client-routes
right after a decommission/bootstrap while gossip/topology settles.
post_client_routes() had no retry, so
test_should_survive_full_node_replacement_through_nlb flaked in CI.

Retry up to 5 attempts with a 1s backoff, but only on 5xx responses -
4xx or other errors still raise immediately. Also log the response
body on failure, which was previously discarded and made past
failures hard to diagnose.

Fixes #931

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../standard/test_client_routes.py            | 42 ++++++++++++++-----
 1 file changed, 31 insertions(+), 11 deletions(-)

diff --git a/tests/integration/standard/test_client_routes.py b/tests/integration/standard/test_client_routes.py
index f365a628f8..4c46428262 100644
--- a/tests/integration/standard/test_client_routes.py
+++ b/tests/integration/standard/test_client_routes.py
@@ -34,6 +34,7 @@
 import uuid
 
 import json as _json
+import urllib.error
 import urllib.request
 
 from cassandra.cluster import Cluster
@@ -232,17 +233,36 @@ def post_client_routes(contact_point, routes):
     url = "http://%s:10000/v2/client-routes" % contact_point
     log.info("Posting %d routes to %s", len(payload), url)
     data = _json.dumps(payload).encode("utf-8")
-    req = urllib.request.Request(
-        url,
-        data=data,
-        headers={
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-        },
-        method="POST",
-    )
-    response = urllib.request.urlopen(req)
-    log.info("Routes posted successfully (status %d)", response.status)
+
+    # A node can still be settling gossip/topology right after a
+    # decommission/bootstrap, making the REST API answer with a transient
+    # 500 for a brief window. Retry bounded 5xx responses; anything else
+    # (4xx, connection errors) is a real bug and should raise immediately.
+    max_attempts = 5
+    for attempt in range(1, max_attempts + 1):
+        req = urllib.request.Request(
+            url,
+            data=data,
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            },
+            method="POST",
+        )
+        try:
+            response = urllib.request.urlopen(req)
+            log.info("Routes posted successfully (status %d)", response.status)
+            return
+        except urllib.error.HTTPError as e:
+            body = e.read().decode("utf-8", "replace")
+            if e.code >= 500 and attempt < max_attempts:
+                log.warning(
+                    "POST %s -> HTTP %d (attempt %d/%d), retrying: %s",
+                    url, e.code, attempt, max_attempts, body)
+                time.sleep(1)
+                continue
+            log.error("POST %s -> HTTP %d: %s", url, e.code, body)
+            raise
 
 
 def get_host_ids_from_cluster(session):