From f28f2e6675ba13318de1b1312055dcb392b793e9 Mon Sep 17 00:00:00 2001 From: DABH Date: Thu, 10 Sep 2026 00:51:22 -0500 Subject: [PATCH] Outlive worker stalls in unfinished-handler tests On the macOS CI runners the test process periodically stalls for 20-30s (junit durations show ~1s tests taking 13-29s). When that happens after the worker has received the first workflow task, the task hits its 10s start-to-close timeout, the transient retry times out too, and the server then rejects the pending UpdateWorkflowExecution because the workflow task attempt reached 3: "Unable to perform workflow execution update due to Workflow Task in failed state." The -no-wait- variants surface the same error as an AssertionError because the RPCError is caught but its status is FAILED_PRECONDITION rather than NOT_FOUND. Signal variants survive the same stall, which is why only the update variants flake. #1824 reordered the cancel and update requests, which is harmless but unrelated. Give these workflows a 60s workflow task timeout so a stall cannot push the task into the failed state, and cancel the background update task left behind by the exceptions sub-test so its eventual RPC error no longer shows up as an unretrieved task exception in later tests. Reproduced deterministically by injecting a 25s synchronous stall into sandbox instance creation: fails before, passes with the longer timeout. --- tests/worker/test_workflow.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/worker/test_workflow.py b/tests/worker/test_workflow.py index 42ba0b69c..eb9ffaf14 100644 --- a/tests/worker/test_workflow.py +++ b/tests/worker/test_workflow.py @@ -6671,6 +6671,10 @@ async def my_signal_WARN_AND_ABANDON(self): await self._do_update_or_signal() +# Outlive CI stalls: two 10s workflow task timeouts make the server fail updates fast +_UNFINISHED_HANDLERS_TASK_TIMEOUT = timedelta(seconds=60) + + async def test_unfinished_update_handler(client: Client): async with new_worker(client, UnfinishedHandlersWarningsWorkflow) as worker: test = _UnfinishedHandlersWarningsTest(client, worker, "update") @@ -6719,16 +6723,20 @@ async def test_unfinished_handlers_cause_exceptions_in_test_suite(self): # If we don't capture warnings then -- since the unfinished handler warning is converted to # an exception in the test suite -- we see WFT failures when we don't wait for handlers. handle: asyncio.Future[WorkflowHandle] = asyncio.Future() - asyncio.create_task( + result_task = asyncio.create_task( self._get_workflow_result( wait_all_handlers_finished=False, handle_future=handle ) ) - await assert_eq_eventually( - True, - partial(self._workflow_task_failed, workflow_id=(await handle).id), - timeout=timedelta(seconds=20), - ) + try: + await assert_eq_eventually( + True, + partial(self._workflow_task_failed, workflow_id=(await handle).id), + timeout=timedelta(seconds=20), + ) + finally: + result_task.cancel() + await asyncio.gather(result_task, return_exceptions=True) async def _workflow_task_failed(self, workflow_id: str) -> bool: resp = await self.client.workflow_service.get_workflow_execution_history( @@ -6771,6 +6779,7 @@ async def _get_workflow_result( arg=wait_all_handlers_finished, id=f"wf-{uuid.uuid4()}", task_queue=self.worker.task_queue, + task_timeout=_UNFINISHED_HANDLERS_TASK_TIMEOUT, ) if handle_future: handle_future.set_result(handle) @@ -6968,6 +6977,7 @@ async def _run_workflow_and_get_warning(self) -> bool: ], id=workflow_id, task_queue=task_queue, + task_timeout=_UNFINISHED_HANDLERS_TASK_TIMEOUT, ) if self.handler_type == "-update-": update_method = (