From 34a221dab91d6e59ec8c5da26ac93c55673bfe07 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Tue, 31 Mar 2026 22:46:26 +0300 Subject: [PATCH 1/6] tests: fix test_roundrobin decommission with dead node Scylla's Raft topology coordinator rejects decommission when there are dead nodes in the cluster. Restart node 3 before decommissioning node 1. --- tests/integration/long/test_loadbalancingpolicies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 072786dc23..bbb8f615bf 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -215,10 +215,10 @@ def test_roundrobin(self): self.coordinator_stats.assert_query_count_equals(2, 6) self.coordinator_stats.assert_query_count_equals(3, 0) + start(3) # Restart before decommission (Raft rejects ops with dead nodes) + self._wait_for_nodes_up([3], cluster) decommission(1) - start(3) self._wait_for_nodes_down([1], cluster) - self._wait_for_nodes_up([3], cluster) self.coordinator_stats.reset_counts() self._query(session, keyspace) From a278230fc7a4fa217091514936d3e3296f3b1c72 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Tue, 31 Mar 2026 22:46:50 +0300 Subject: [PATCH 2/6] tests: fix test_roundrobin_two_dcs bootstrap with dead node Scylla's Raft topology coordinator rejects bootstrap when there are dead nodes in the cluster. Bootstrap node 5 before force-stopping node 1. --- tests/integration/long/test_loadbalancingpolicies.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index bbb8f615bf..9cbf30991b 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -244,14 +244,13 @@ def test_roundrobin_two_dcs(self): self.coordinator_stats.assert_query_count_equals(3, 3) self.coordinator_stats.assert_query_count_equals(4, 3) + bootstrap(5, 'dc3') # Bootstrap before force_stop (Raft rejects ops with dead nodes) + self._wait_for_nodes_up([5], cluster) force_stop(1) - bootstrap(5, 'dc3') # reset control connection self._insert(session, keyspace, count=1000) - self._wait_for_nodes_up([5], cluster) - self.coordinator_stats.reset_counts() self._query(session, keyspace) From d06113b38084b6ff9628320a93316a59d65afb06 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Tue, 31 Mar 2026 22:47:19 +0300 Subject: [PATCH 3/6] tests: fix test_roundrobin_two_dcs_2 bootstrap with dead node Scylla's Raft topology coordinator rejects bootstrap when there are dead nodes in the cluster. Bootstrap node 5 before force-stopping node 1. --- tests/integration/long/test_loadbalancingpolicies.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 9cbf30991b..18b1eee694 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -277,14 +277,13 @@ def test_roundrobin_two_dcs_2(self): self.coordinator_stats.assert_query_count_equals(3, 3) self.coordinator_stats.assert_query_count_equals(4, 3) + bootstrap(5, 'dc1') # Bootstrap before force_stop (Raft rejects ops with dead nodes) + self._wait_for_nodes_up([5], cluster) force_stop(1) - bootstrap(5, 'dc1') # reset control connection self._insert(session, keyspace, count=1000) - self._wait_for_nodes_up([5], cluster) - self.coordinator_stats.reset_counts() self._query(session, keyspace) From c6845eeea907da87eaa69bd8e673bce7a85a79e3 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Tue, 31 Mar 2026 22:48:54 +0300 Subject: [PATCH 4/6] tests: fix test_token_aware_with_rf_2 for Scylla shard-aware routing Scylla's shard-aware routing may distribute TokenAwarePolicy queries across both replicas instead of always picking the first one. Assert that the total query count across both replicas equals 12. --- tests/integration/long/test_loadbalancingpolicies.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 18b1eee694..216f177e61 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -521,8 +521,9 @@ def test_token_aware_with_rf_2(self, use_prepared=False): self._query(session, keyspace) self.coordinator_stats.assert_query_count_equals(1, 0) - self.coordinator_stats.assert_query_count_equals(2, 12) - self.coordinator_stats.assert_query_count_equals(3, 0) + # Scylla may distribute queries across both replicas with shard-aware routing + queried = self.coordinator_stats.get_query_count(2) + self.coordinator_stats.get_query_count(3) + assert queried == 12, "Expected 12 queries to replicas, got %d" % queried self.coordinator_stats.reset_counts() stop(2) From 1581ac013b8d82464c2cd93c48d65dea73c86752 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Sun, 12 Apr 2026 13:21:49 +0300 Subject: [PATCH 5/6] tests: fix test_dc_aware_roundrobin_one_remote_host decommission order Scylla's Raft topology coordinator can fail when decommissioning with only 2 voters. Reorder decommissions so node 1 is decommissioned while 3 Raft voters remain (nodes 1, 2, 5), before decommissioning node 5. --- .../integration/long/test_loadbalancingpolicies.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 216f177e61..3406e653a7 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -372,23 +372,25 @@ def test_dc_aware_roundrobin_one_remote_host(self): responses.add(self.coordinator_stats.get_query_count(node)) assert set([0, 0, 12]) == responses + # Decommission node 1 while 3 Raft voters remain (nodes 1, 2, 5). + # Doing this later (with only 2 voters) can cause Raft issues. self.coordinator_stats.reset_counts() - decommission(5) - self._wait_for_nodes_down([5]) + decommission(1) + self._wait_for_nodes_down([1]) self._query(session, keyspace) + self.coordinator_stats.assert_query_count_equals(1, 0) self.coordinator_stats.assert_query_count_equals(3, 0) self.coordinator_stats.assert_query_count_equals(4, 0) - self.coordinator_stats.assert_query_count_equals(5, 0) responses = set() - for node in [1, 2]: + for node in [2, 5]: responses.add(self.coordinator_stats.get_query_count(node)) assert set([0, 12]) == responses self.coordinator_stats.reset_counts() - decommission(1) - self._wait_for_nodes_down([1]) + decommission(5) + self._wait_for_nodes_down([5]) self._query(session, keyspace) From 3a3b851779ec8294533ea2655bd1dc5533adf906 Mon Sep 17 00:00:00 2001 From: Yaniv Michael Kaul Date: Sun, 12 Apr 2026 13:21:55 +0300 Subject: [PATCH 6/6] tests: add remove_cluster() to prevent stale decommissioned node state test_token_aware_composite_key and test_token_aware_with_local_table called use_singledc() without remove_cluster(), reusing the cluster left by the previous decommission-heavy test. Decommissioned nodes refused to restart with 'This node was decommissioned and will not rejoin the ring'. Adding remove_cluster() forces a fresh cluster. --- tests/integration/long/test_loadbalancingpolicies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 3406e653a7..eba5b4a348 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -479,6 +479,7 @@ def token_aware(self, keyspace, use_prepared=False): self.coordinator_stats.assert_query_count_equals(2, 0) def test_token_aware_composite_key(self): + remove_cluster() use_singledc() keyspace = 'test_token_aware_composite_key' table = 'composite' @@ -538,6 +539,7 @@ def test_token_aware_with_rf_2(self, use_prepared=False): self.coordinator_stats.assert_query_count_equals(3, 12) def test_token_aware_with_local_table(self): + remove_cluster() use_singledc() cluster, session = self._cluster_session_with_lbp(TokenAwarePolicy(RoundRobinPolicy())) self.addCleanup(cluster.shutdown)