From cccbaf85e9fad11db06f6e86a2b7031213f55945 Mon Sep 17 00:00:00 2001 From: DABH Date: Thu, 10 Sep 2026 00:22:49 -0500 Subject: [PATCH] Use retrying client for worker deployment RPCs in tests The frontend recycles client connections after frontend.keepAliveMaxConnectionAge (5 minutes, jittered) by sending GOAWAY. While worker long polls keep the retiring connection open, the first request queued on it fails inside hyper with "operation was canceled", which tonic surfaces as gRPC CANCELLED. The sdk-core retry client recognizes this transport-sourced CANCELLED and retries it, but the deployment test helpers called the raw service stubs with the default retry=False, so once per connection lifetime a SetWorkerDeploymentCurrentVersion or SetWorkerDeploymentRampingVersion call could fail with RPCError: (1, 'operation was canceled', b''). Route the helpers through the retrying client, as the high-level Client does for its own RPCs. --- tests/worker/test_worker.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/worker/test_worker.py b/tests/worker/test_worker.py index 57614c21e..7aa666cd4 100644 --- a/tests/worker/test_worker.py +++ b/tests/worker/test_worker.py @@ -1281,7 +1281,8 @@ async def mk_call() -> DescribeWorkerDeploymentResponse: DescribeWorkerDeploymentRequest( namespace=client.namespace, deployment_name=version.deployment_name, - ) + ), + retry=True, ) except RPCError: # Expected @@ -1304,7 +1305,8 @@ async def set_current_deployment_version( deployment_name=version.deployment_name, version=version.to_canonical_string(), conflict_token=conflict_token, - ) + ), + retry=True, ) @@ -1321,7 +1323,8 @@ async def set_ramping_version( version=version.to_canonical_string(), conflict_token=conflict_token, percentage=percentage, - ) + ), + retry=True, ) return response @@ -1341,7 +1344,8 @@ async def check() -> bool: DescribeWorkerDeploymentRequest( namespace=client.namespace, deployment_name=deployment_name, - ) + ), + retry=True, ) routing_config = resp.worker_deployment_info.routing_config if (