From 086b3839624a112b0d71703200df033bc335c57d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:17:19 +0000 Subject: [PATCH 01/12] Negotiate replay-safe worker task offers and preserve uncertain acceptances --- docs/portal/build/workers.md | 8 + docs/protocol.md | 10 + src/Client.php | 52 ++++- src/Exception/TaskDeliveryUncertain.php | 23 +++ src/Transport/PollTransport.php | 10 +- src/Worker.php | 2 + src/Worker/PollResponse.php | 5 +- src/Worker/TaskDelivery.php | 179 ++++++++++++++++ src/Worker/TaskDeliveryAttempt.php | 53 +++++ tests/ClientContractTest.php | 2 + tests/WorkerAuthoringTest.php | 2 + tests/WorkerPollTransportTest.php | 21 ++ tests/WorkerTaskDeliveryTest.php | 259 ++++++++++++++++++++++++ 13 files changed, 615 insertions(+), 11 deletions(-) create mode 100644 src/Exception/TaskDeliveryUncertain.php create mode 100644 src/Worker/TaskDelivery.php create mode 100644 src/Worker/TaskDeliveryAttempt.php create mode 100644 tests/WorkerTaskDeliveryTest.php diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index 41e9e64b..ca88fbc5 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -99,6 +99,14 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n ### Poll timeouts and early disconnects +When the Server advertises `task_delivery`, the managed worker negotiates +offer/accept delivery. Receiving an offer does not reserve a task or start an +activity attempt. The SDK accepts the selected offer before running a handler. +If that response is lost, it retries the same packet and poll identity until +the Server confirms the claim, expiry, or loss of registration authority. +An older node returning an error cannot switch that poll to legacy delivery. +Servers without this capability continue using the existing poll protocol. + With the default cURL transport, supporting Servers send progress during idle worker polls. Progress keeps an idle connection active while the complete request remains bounded to the requested wait plus five seconds. Ordinary polls diff --git a/docs/protocol.md b/docs/protocol.md index 6ce822cb..7814f0c1 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -198,6 +198,16 @@ may have committed a claim before the connection disappeared, so `task=null` does not establish that no lease exists. This does not make arbitrary database errors or unrelated structured HTTP 500/503 responses retryable. +With negotiated `task_delivery`, worker polls use `/offers`. An offer alone +grants no execution authority. Positive acceptance includes the original +registration token and selected offer packet; its request body remains fixed +through uncertain replies. Managed workers retain this state across client +clones and honor the normal retry backoff. If using the low-level +`poll*TaskResponse()` methods, catch `TaskDeliveryUncertain` and retry with its +`details['poll_request_id']` on the same Client. An ordinary empty response to +an acceptance does not prove that no claim committed. A confirmed offer expiry +permits a new poll; lost registration authority stops that worker. + Before replaying or completing a workflow task, the managed worker requires a successful lease-renewal response whose task ID, attempt, and lease owner match the current claim. A typed `renewed=false`, `retryable=true` response is retried diff --git a/src/Client.php b/src/Client.php index d4d3aa4e..cf5ba4a1 100644 --- a/src/Client.php +++ b/src/Client.php @@ -62,6 +62,9 @@ use DurableWorkflow\Worker\WorkerSessionOptions; use DurableWorkflow\Worker\ActivityLeaseBinding; use DurableWorkflow\Worker\ActivityLeaseProcess; +use DurableWorkflow\Worker\TaskDelivery; +use DurableWorkflow\Worker\TaskDeliveryAttempt; +use DurableWorkflow\Transport\PollTransport; use InvalidArgumentException; /** Synchronous control-plane and worker-plane client for the standalone server. */ @@ -81,6 +84,7 @@ final class Client implements WorkflowClientInterface private FailureReportLimiter $failureReports; private bool $boundedWorkerRequests = false; private ActivityLeaseBinding $activityLeaseBinding; + private TaskDelivery $taskDelivery; private ?string $activityLeaseBootstrap = null; private ?ExecutionTelemetry $executionTelemetry = null; private bool $observabilityContextSupported = false; @@ -143,6 +147,7 @@ public function __construct( $this->payloadUploads = new RuntimePayloadUploads($this->transport, $this->baseUri); $this->failureReports = new FailureReportLimiter(new FailureReportLimits(), $this->transport); $this->activityLeaseBinding = new ActivityLeaseBinding(); + $this->taskDelivery = new TaskDelivery(); } /** Narrow the encoded failure-report request budget for an application transport. */ @@ -1576,6 +1581,7 @@ public function registerWorker( 'memo_upserts', 'typed_search_attributes', 'durable_selection', + TaskDelivery::CAPABILITY, ], int $maxConcurrentWorkflowTasks = 1, int $maxConcurrentActivityTasks = 1, @@ -1585,6 +1591,10 @@ public function registerWorker( int $maxConcurrentWorkerSessions = 10, ?array $workflowDefinitionFingerprints = null, ): array { + if ($this->workerProtocolVersion !== '1.20') { + $capabilities = array_values(array_filter($capabilities, + static fn (string $capability): bool => $capability !== TaskDelivery::CAPABILITY)); + } if (in_array('message_streams', $capabilities, true) && !Version::supportsMessageStreams()) { throw new InvalidArgumentException('Message streams require worker protocol 1.15 or newer.'); } @@ -1606,7 +1616,7 @@ public function registerWorker( } } - return $this->worker('POST', '/worker/register', $this->withoutNulls([ + $response = $this->worker('POST', '/worker/register', $this->withoutNulls([ 'worker_id' => $workerId, 'task_queue' => $taskQueue, 'runtime' => 'php', @@ -1621,7 +1631,11 @@ public function registerWorker( 'max_concurrent_activity_tasks' => $maxConcurrentActivityTasks, 'max_concurrent_worker_sessions' => $maxConcurrentWorkerSessions, 'build_id' => $buildId, + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ])); + $this->taskDelivery->registered($workerId, $taskQueue, $this->workerProtocolVersion, $capabilities, $response); + + return $response; } /** @return array */ @@ -1678,6 +1692,7 @@ public function heartbeatWorker(string $workerId, array $taskSlots = [], ?Reques public function deregisterWorker(string $workerId): void { $this->control('DELETE', '/workers/'.$this->segment($workerId)); + $this->taskDelivery->forget($workerId); } /** @return array{worker_id: string, outcome: string, recovered_workflow_task_count: int} */ @@ -1702,6 +1717,8 @@ public function deregisterWorkerRegistration(string $workerId, ?string $registra 'invalid_worker_deregistration_receipt', $response); } + $this->taskDelivery->forget($workerId); + /** @var array{worker_id: string, outcome: string, recovered_workflow_task_count: int} $response */ return $response; } @@ -2592,7 +2609,12 @@ private function pollTaskResponse(string $path, array $body): array $retryTransientFailure = true; while (true) { try { - return $this->worker('POST', $path, $body); + return $this->taskDelivery->poll($path, $body, + fn (string $wirePath, array $wireBody, ?TaskDeliveryAttempt $selected): array => + $this->request('POST', $wirePath, true, $wireBody, + responseValidator: static fn (array $response) => + TaskDelivery::assertResponse($response, $wireBody, $wirePath, $selected))) + ?? $this->worker('POST', $path, $body); } catch (ServerException $exception) { if ($retryTransientFailure && $exception->isTransientConnectionFailure()) { $retryTransientFailure = false; @@ -2659,7 +2681,7 @@ private function sendTransportRequest(string $method, string $path, bool $worker } /** @return array */ - private function requestHeaders(bool $worker): array + private function requestHeaders(bool $worker, string $path = ''): array { $headers = [ 'Accept' => 'application/json', @@ -2671,6 +2693,9 @@ private function requestHeaders(bool $worker): array if ($this->authentication !== null) { $headers = array_merge($headers, $this->authentication->headers($worker)); } + if ($worker && PollTransport::taskKind('POST', $path) !== null && str_ends_with($path, '/offers')) { + $headers[TaskDelivery::HEADER] = TaskDelivery::VERSION; + } return $headers; } @@ -2678,6 +2703,7 @@ private function requestHeaders(bool $worker): array /** * @param array|null $body * @param (\Closure(array): list>)|null $payloadPaths + * @param (\Closure(array): void)|null $responseValidator * @return array */ private function request( @@ -2689,11 +2715,13 @@ private function request( ?RequestBudget $budget = null, ?\Closure $payloadPaths = null, ?string $requestId = null, + ?\Closure $responseValidator = null, ): array { - $headers = $this->requestHeaders($worker); + $headers = $this->requestHeaders($worker, $path); + $logicalPath = PollTransport::logicalPath($path); try { $callerBudget = $budget; - $poll = str_ends_with($path, '/poll') && is_int($body['timeout_seconds'] ?? null); + $poll = PollTransport::taskKind($method, $path) !== null && is_int($body['timeout_seconds'] ?? null); $pollSeconds = $poll ? max(0, min(60, $body['timeout_seconds'])) : 0; $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; if ($body !== null) { @@ -2703,7 +2731,7 @@ private function request( $body = $this->failureReports->request($path, $body); } $activityLease = $worker ? $this->activityLeaseBinding->current() : null; - if ($activityLease !== null && $activityLease->governs($path)) { + if ($activityLease !== null && $activityLease->governs($logicalPath)) { // Uploading an external result is not publication authority. // Check again before its final Worker POST and bound the send // by the latest acknowledged grant, not an old cached lease. @@ -2713,21 +2741,27 @@ private function request( $response = $this->sendTransportRequest($method, $path, $worker, $headers, $body, $budget); $budget?->remainingSeconds(); + if ($responseValidator !== null) { + if (!is_array($response) || array_is_list($response)) { + throw new ServerException('The task-delivery response is invalid.', 200, 'invalid_task_delivery_response'); + } + $responseValidator($response); + } // Control I/O and supervised payload I/O have separate bounds. // Explicit caller/cleanup budgets still cover their entire request. $responseBudget = $budget !== null && $poll && $callerBudget === null ? new RequestBudget(5) : $budget; - $canStreamBounded = $worker && ($this->boundedWorkerRequests || $path === '/worker/query-tasks/poll') + $canStreamBounded = $worker && ($this->boundedWorkerRequests || $logicalPath === '/worker/query-tasks/poll') && $this->transport instanceof \DurableWorkflow\Transport\BoundedProgressPayloadTransport && $this->transport instanceof BoundedTransport && $this->transport->supportsBoundedRequests(); $observerBudget = $canStreamBounded && $callerBudget === null ? null : $responseBudget; $progress = $worker && $this->workerPayloadObserver !== null && is_array($response) && !array_is_list($response) - ? ($this->workerPayloadObserver)($path, $body, $response, $observerBudget) : null; + ? ($this->workerPayloadObserver)($logicalPath, $body, $response, $observerBudget) : null; $streamBounded = $canStreamBounded && $progress !== null; if ($streamBounded && $callerBudget === null) { $responseBudget = null; } $result = is_array($response) && !array_is_list($response) - ? (new RuntimePayloads($this->transport, $this->baseUri, $headers, $this->maxExternalPayloadBytes, $responseBudget, $progress, $streamBounded))->response($response, $path, $worker, $payloadPaths?->__invoke($response)) + ? (new RuntimePayloads($this->transport, $this->baseUri, $headers, $this->maxExternalPayloadBytes, $responseBudget, $progress, $streamBounded))->response($response, $logicalPath, $worker, $payloadPaths?->__invoke($response)) : []; $responseBudget?->remainingSeconds(); diff --git a/src/Exception/TaskDeliveryUncertain.php b/src/Exception/TaskDeliveryUncertain.php new file mode 100644 index 00000000..035f351c --- /dev/null +++ b/src/Exception/TaskDeliveryUncertain.php @@ -0,0 +1,23 @@ + null, 'poll_request_id' => $pollRequestId, + 'retry_same_poll_request_id' => true, + ...($previous instanceof ServerException && is_int($previous->details['retry_after_seconds'] ?? null) + && $previous->details['retry_after_seconds'] > 0 + ? ['retry_after_seconds' => $previous->details['retry_after_seconds']] : []), + ], $previous); + } +} diff --git a/src/Transport/PollTransport.php b/src/Transport/PollTransport.php index 4aaf2088..3f719441 100644 --- a/src/Transport/PollTransport.php +++ b/src/Transport/PollTransport.php @@ -19,7 +19,8 @@ public static function taskKind(string $method, string $uri): ?string $path = parse_url($uri, PHP_URL_PATH); if (strtoupper($method) === 'POST' && is_string($path)) { foreach (self::KINDS as $kind) { - if (str_ends_with($path, '/worker/'.$kind.'-tasks/poll')) { + if (str_ends_with($path, '/worker/'.$kind.'-tasks/poll') + || str_ends_with($path, '/worker/'.$kind.'-tasks/offers')) { return $kind; } } @@ -28,6 +29,13 @@ public static function taskKind(string $method, string $uri): ?string return null; } + /** Payload observers and claim supervision use the original logical poll. */ + public static function logicalPath(string $path): string + { + return self::taskKind('POST', $path) !== null && str_ends_with($path, '/offers') + ? substr($path, 0, -7).'/poll' : $path; + } + /** @return array{status: int, headers: array, body: array} */ public static function decode(string $rawBody, int $outerStatus): array { diff --git a/src/Worker.php b/src/Worker.php index 57192c9e..92995267 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -656,6 +656,7 @@ private function registerWithRetry(): ?array 'worker_sessions', 'sticky_execution', 'cross_kind_poll_wake', + ...($this->client->workerProtocolVersion === '1.20' ? [\DurableWorkflow\Worker\TaskDelivery::CAPABILITY] : []), ...($this->enableCooperativeCancellation ? ['cooperative_cancellation', 'nexus_cooperative_cancellation'] : []), ...($this->enablePreparedLocalActivities ? ['prepared_local_activities'] : []), ...($this->preparedLocalActivityGroupsSupported ? ['prepared_local_activity_groups'] : []), @@ -929,6 +930,7 @@ private function pollWithRetry(string $taskKind, \Closure $poll, int $requestedT } if (!$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() + && !$exception instanceof \DurableWorkflow\Exception\TaskDeliveryUncertain && !PollResponse::isTransientPayloadFailure($exception) && !$databaseUnavailable && !$storageAdmission) { $requestId = 'php-'.$taskKind.'-poll-'.bin2hex(random_bytes(16)); diff --git a/src/Worker/PollResponse.php b/src/Worker/PollResponse.php index e09477df..62a4c261 100644 --- a/src/Worker/PollResponse.php +++ b/src/Worker/PollResponse.php @@ -6,6 +6,7 @@ use DurableWorkflow\Exception\ExternalPayloadException; use DurableWorkflow\Exception\ServerException; +use DurableWorkflow\Exception\TaskDeliveryUncertain; /** Classifies full worker poll response envelopes using protocol fields. */ final class PollResponse @@ -23,6 +24,7 @@ final class PollResponse 'worker_heartbeat_stale', 'worker_draining', 'worker_stopped', + 'worker_registration_lost_authority', ]; /** @param array $response */ @@ -42,7 +44,8 @@ public static function isTerminal(array $response): bool */ public static function isTransientFailure(ServerException $exception): bool { - if ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() + if ($exception instanceof TaskDeliveryUncertain + || $exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() || self::isTransientPayloadFailure($exception)) { return true; } diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php new file mode 100644 index 00000000..9ab67953 --- /dev/null +++ b/src/Worker/TaskDelivery.php @@ -0,0 +1,179 @@ + */ + private array $registrations = []; + /** @var array */ + private array $pending = []; + + /** + * @param list $capabilities + * @param array $response + */ + public function registered(string $workerId, string $queue, string $protocol, array $capabilities, array $response): void + { + $server = $response['server_capabilities'] ?? []; + $feature = is_array($server) ? ($server['task_delivery'] ?? null) : null; + if ($protocol !== '1.20' || !in_array(self::CAPABILITY, $capabilities, true) + || $feature === null || (is_array($feature) && ($feature['supported'] ?? null) === false)) { + foreach ($this->pending as $attempt) { + if ($attempt->workerId === $workerId) { + throw new ServerException('An unresolved delivery cannot change transport modes.', 201, 'invalid_task_delivery_capability'); + } + } + $this->forget($workerId); + return; + } + $token = $response['registration_token'] ?? null; + if (!is_array($feature) || array_is_list($feature) || ($feature['supported'] ?? null) !== true + || ($feature['schema'] ?? null) !== self::SCHEMA || ($feature['version'] ?? null) !== self::VERSION + || ($feature['header'] ?? null) !== self::HEADER || ($feature['worker_capability'] ?? null) !== self::CAPABILITY + || ($feature['minimum_protocol_version'] ?? null) !== '1.20' || ($feature['endpoint_suffix'] ?? null) !== '/offers' + || ($feature['unaccepted_offer_reserves_task'] ?? null) !== false + || !is_int($feature['acceptance_timeout_seconds'] ?? null) + || $feature['acceptance_timeout_seconds'] < 1 || $feature['acceptance_timeout_seconds'] > 10 + || !is_string($token) || preg_match('/^[a-f0-9]{32}$/D', $token) !== 1) { + throw new ServerException('The task-delivery registration contract is invalid.', 201, 'invalid_task_delivery_capability'); + } + if (($this->registrations[$workerId] ?? null) !== ['queue' => $queue, 'token' => $token]) { + $this->forget($workerId); + } + $this->registrations[$workerId] = ['queue' => $queue, 'token' => $token]; + } + + /** + * @param array $body + * @param Closure(string, array, ?TaskDeliveryAttempt): array $send + * @return array|null Null means negotiation never selected this mode. + */ + public function poll(string $path, array $body, Closure $send): ?array + { + $workerId = (string) $body['worker_id']; + $registration = $this->registrations[$workerId] ?? null; + if ($registration === null || $registration['queue'] !== ($body['task_queue'] ?? null)) { + return null; + } + $key = hash('sha256', json_encode([$path, $workerId, $body['task_queue'], $body['poll_request_id']], JSON_THROW_ON_ERROR)); + $selected = $this->pending[$key] ?? null; + try { + if ($selected === null) { + $wirePath = substr($path, 0, -5).'/offers'; + $body['registration_token'] = $registration['token']; + $response = $send($wirePath, $body, null); + if (!array_key_exists('task_offer', $response)) { + return $response; + } + $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, + (string) PollTransport::taskKind('POST', $path)); + // Concurrent callers for one logical poll select only its first packet. + $selected = $this->pending[$key] ??= $candidate; + } + $response = $send($selected->path, $selected->requestBody(), $selected); + if ($response['task'] === null && !PollResponse::isTerminal($response)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); + } + $this->resolved($key, $selected); + + return $response; + } catch (ServerException $exception) { + if ($this->definitiveRefusal($exception)) { + $this->resolved($key, $selected); + $response = $exception->details ?? []; + if ($exception->reason === 'worker_registration_lost_authority') { + $this->forget($workerId); + } else { + $response['poll_status'] = 'empty'; + } + + return $response; + } + if ($exception instanceof TaskDeliveryUncertain) { + throw $exception; + } + if (($selected !== null && ($exception->status >= 500 || $exception->status === 409 || $exception->status === 429 + || $exception->reason === 'invalid_task_delivery_response' + || in_array($exception->status, [0, 404, 405], true))) + || ($selected === null && in_array($exception->status, [404, 405], true))) { + if (PollResponse::isTransientPayloadFailure($exception)) { + throw $exception; + } + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); + } + + throw $exception; + } + } + + /** + * Validate before payload hydration can initialize a claim supervisor. + * @param array $response + * @param array $body + */ + public static function assertResponse(array $response, array $body, string $path, ?TaskDeliveryAttempt $selected): void + { + $task = $response['task'] ?? null; + $kind = PollTransport::taskKind('POST', $path); + $idField = match ($kind) { + 'query' => 'query_task_id', 'update-validation' => 'update_validation_task_id', default => 'task_id', + }; + if (!array_key_exists('task', $response) + || (($selected !== null || $task !== null) && array_key_exists('task_offer', $response)) + || (($response['poll_status'] ?? null) === 'offered' && !array_key_exists('task_offer', $response)) + || ($task !== null && (!is_array($task) || array_is_list($task) + || !is_string($task[$idField] ?? null) || $task[$idField] === '' + || ($task['lease_owner'] ?? null) !== $body['worker_id'] + || ($selected !== null && $task[$idField] !== $selected->taskId) + || (isset($task['task_kind']) && $task['task_kind'] !== str_replace('-', '_', (string) $kind))))) { + throw new ServerException('The task-delivery response is invalid.', 200, 'invalid_task_delivery_response'); + } + } + + public function forget(string $workerId): void + { + unset($this->registrations[$workerId]); + foreach ($this->pending as $key => $attempt) { + if ($attempt->workerId === $workerId) { + unset($this->pending[$key]); + } + } + } + + /** @return array */ + public function __debugInfo(): array + { + return ['registered_workers' => count($this->registrations), 'unresolved_deliveries' => count($this->pending)]; + } + + private function definitiveRefusal(ServerException $exception): bool + { + $response = $exception->details; + + return $exception->status === 409 && $response !== null && !array_is_list($response) + && array_key_exists('task', $response) && $response['task'] === null + && ($response['reason'] ?? null) === $exception->reason + && in_array($exception->reason, ['task_offer_expired', 'worker_registration_lost_authority'], true); + } + + private function resolved(string $key, ?TaskDeliveryAttempt $selected): void + { + if (($this->pending[$key] ?? null) === $selected) { + unset($this->pending[$key]); + } + } +} diff --git a/src/Worker/TaskDeliveryAttempt.php b/src/Worker/TaskDeliveryAttempt.php new file mode 100644 index 00000000..64cff2ca --- /dev/null +++ b/src/Worker/TaskDeliveryAttempt.php @@ -0,0 +1,53 @@ + $body */ + private function __construct( + public readonly string $workerId, + public readonly string $path, + public readonly string $taskId, + private readonly array $body, + ) { + } + + /** + * @param array $body + * @param array $response + */ + public static function fromResponse(string $path, array $body, array $response, string $kind): self + { + $offer = $response['task_offer'] ?? null; + if (!array_key_exists('task', $response) || $response['task'] !== null || ($response['poll_status'] ?? null) !== 'offered' + || !is_array($offer) || array_is_list($offer) + || ($offer['schema'] ?? null) !== TaskDelivery::SCHEMA + || ($offer['task_kind'] ?? null) !== str_replace('-', '_', $kind) + || !is_string($offer['task_id'] ?? null) || $offer['task_id'] === '' + || !is_string($offer['token'] ?? null) || $offer['token'] === '' || strlen($offer['token']) > 8192 + || !is_int($offer['expires_at_ms'] ?? null) || $offer['expires_at_ms'] < 1) { + throw new ServerException('The server returned an invalid task offer.', 200, 'invalid_task_delivery_offer'); + } + + return new self((string) $body['worker_id'], $path, $offer['task_id'], + array_replace($body, ['task_offer_token' => $offer['token'], 'timeout_seconds' => 0])); + } + + /** @return array */ + public function requestBody(): array + { + return $this->body; + } + + /** @return array */ + public function __debugInfo(): array + { + return ['worker_id' => $this->workerId, 'path' => $this->path, 'task_id' => $this->taskId]; + } +} diff --git a/tests/ClientContractTest.php b/tests/ClientContractTest.php index 439a9c90..eef3ec12 100644 --- a/tests/ClientContractTest.php +++ b/tests/ClientContractTest.php @@ -119,6 +119,7 @@ public function testDefaultWorkerRegistrationAdvertisesTheCompletePhpCapabilityS 'memo_upserts', 'typed_search_attributes', 'durable_selection', + 'task_delivery_offers', ], 'capability_manifest' => [ 'local_activities' => [ @@ -140,6 +141,7 @@ public function testDefaultWorkerRegistrationAdvertisesTheCompletePhpCapabilityS 'max_concurrent_workflow_tasks' => 1, 'max_concurrent_activity_tasks' => 1, 'max_concurrent_worker_sessions' => 10, + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ], ], $transport->requests[0]); } diff --git a/tests/WorkerAuthoringTest.php b/tests/WorkerAuthoringTest.php index b8918bfe..930c2af6 100644 --- a/tests/WorkerAuthoringTest.php +++ b/tests/WorkerAuthoringTest.php @@ -182,6 +182,7 @@ public function testHighLevelWorkerAdvertisesTheCompleteManagedCapabilitySet(): 'worker_sessions', 'sticky_execution', 'cross_kind_poll_wake', + 'task_delivery_offers', ], 'capability_manifest' => [ 'local_activities' => [ @@ -204,6 +205,7 @@ public function testHighLevelWorkerAdvertisesTheCompleteManagedCapabilitySet(): 'max_concurrent_activity_tasks' => 1, 'max_concurrent_worker_sessions' => 10, 'build_id' => 'release-a', + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ], ], $transport->requests[0]); } diff --git a/tests/WorkerPollTransportTest.php b/tests/WorkerPollTransportTest.php index eda12b19..2ecf2df0 100644 --- a/tests/WorkerPollTransportTest.php +++ b/tests/WorkerPollTransportTest.php @@ -51,6 +51,27 @@ public static function pollKinds(): array return [['workflow'], ['activity'], ['query'], ['update-validation']]; } + #[DataProvider('pollKinds')] + public function testOfferAndAcceptanceKeepProgressNegotiationAndTheirAbsoluteBudgets(string $kind): void + { + $calls = []; + $http = new GuzzleClient(['handler' => static function (RequestInterface $request, array $options) use (&$calls) { + $calls[] = [$request, $options]; + return Create::promiseFor(new Response(200, ['Content-Type' => self::MEDIA_TYPE], self::frame(200))); + }]); + $transport = new Psr18Transport($http); + $uri = 'https://server.example/api/worker/'.$kind.'-tasks/offers'; + $transport->send('POST', $uri, [], ['timeout_seconds' => 4, 'poll_request_id' => 'same-poll']); + $transport->send('POST', $uri, [], ['timeout_seconds' => 0, 'poll_request_id' => 'same-poll', 'task_offer_token' => 'same-packet']); + self::assertSame(9, $calls[0][1]['timeout']); + self::assertSame(5, $calls[1][1]['timeout']); + foreach ($calls as [$request, $options]) { + self::assertSame(self::MEDIA_TYPE.', application/json;q=0.9', $request->getHeaderLine('Accept')); + self::assertFalse($options['stream']); + } + self::assertSame('same-packet', json_decode((string) $calls[1][0]->getBody(), true)['task_offer_token']); + } + public function testAShorterInjectedGuzzleDeadlineIsPreserved(): void { $observed = null; diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php new file mode 100644 index 00000000..66a81788 --- /dev/null +++ b/tests/WorkerTaskDeliveryTest.php @@ -0,0 +1,259 @@ +withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?\Closure { + if ($response['task'] ?? null) { + $observed[] = $path; + } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed($kind), self::poll($client, $kind)); + self::assertCount(3, $transport->requests); + self::assertSame(['/worker/'.$kind.'-tasks/poll'], $observed); + $initial = $transport->requests[1]; + $accept = $transport->requests[2]; + self::assertStringEndsWith('/worker/'.$kind.'-tasks/offers', $initial['uri']); + self::assertSame($initial['uri'], $accept['uri']); + self::assertSame(TaskDelivery::VERSION, $initial['headers'][TaskDelivery::HEADER]); + self::assertSame($initial['headers'], $accept['headers']); + self::assertSame(4, $initial['body']['timeout_seconds']); + self::assertSame(array_replace($initial['body'], ['timeout_seconds' => 0, 'task_offer_token' => 'opaque-first-packet']), $accept['body']); + self::assertSame(str_repeat('a', 32), $initial['body']['registration_token']); + self::assertSame(getmypid(), $transport->requests[0]['body']['process_metrics']['process_id']); + self::assertArrayNotHasKey(TaskDelivery::HEADER, $transport->requests[0]['headers']); + } + + #[DataProvider('uncertainOutcomes')] + public function testUncertainRepliesRetryOnlyTheFirstSelectedPacket(TransportException|array|null $outcome): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), $outcome, self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'workflow'); + self::fail('The uncertain acceptance must not authorize execution or start another poll.'); + } catch (TaskDeliveryUncertain $error) { + self::assertTrue(PollResponse::isTransientFailure($error)); + self::assertSame('fixed-poll', $error->details['poll_request_id']); + self::assertArrayNotHasKey('task_offer_token', $error->details); + } + // A clone and changed wait/page inputs cannot replace an unresolved packet. + $clone = $client->withWorkerPollCutoffObserver(static function (): void {}); + self::assertSame(self::claimed('workflow'), $clone->pollWorkflowTaskResponse('worker', 'orders', 19, 'fixed-poll', 30)); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertSame('opaque-first-packet', $transport->requests[3]['body']['task_offer_token']); + } + + public static function uncertainOutcomes(): iterable + { + yield 'lost response' => [new TransportException('Connection reset.', transientConnectionFailure: true)]; + yield 'gateway timeout' => [new TransportException('Gateway timeout.', 504)]; + yield 'older node' => [new TransportException('Not found.', 404)]; + yield 'structured backend failure' => [new TransportException('Backend unavailable.', 503, + ['task' => null, 'reason' => 'backend_unavailable', 'retryable' => true])]; + yield 'lock pressure' => [new TransportException('Lock pressure.', 503, + ['task' => null, 'reason' => 'backend_lock_pressure', 'poll_status' => 'backend_lock_pressure', 'retry_after_seconds' => 1])]; + yield 'packet unavailable on another node' => [new TransportException('Unavailable.', 409, + ['task' => null, 'reason' => 'task_offer_unavailable'])]; + yield 'empty is not proof of no claim' => [['task' => null, 'poll_status' => 'empty']]; + yield 'malformed response' => [null]; + yield 'wrong task' => [['task' => ['task_id' => 'other', 'lease_owner' => 'worker']]]; + yield 'wrong owner' => [['task' => ['task_id' => 'task', 'lease_owner' => 'other']]]; + } + + public function testAnOlderNodeCannotDowngradeTheNegotiatedEndpoint(): void + { + $transport = new FakeTransport([self::registration(), new TransportException('Not found.', 404), ['task' => null]]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'activity'); + self::fail('Negotiated delivery cannot fall back to a legacy immediate claim.'); + } catch (TaskDeliveryUncertain) { + self::assertSame(['task' => null], self::poll($client, 'activity')); + } + self::assertSame($transport->requests[1], $transport->requests[2]); + self::assertStringEndsWith('/activity-tasks/offers', $transport->requests[2]['uri']); + } + + public function testAConfirmedExpiryReleasesTheSelectedPacket(): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Expired.', 409, ['task' => null, 'poll_status' => 'task_offer_expired', 'reason' => 'task_offer_expired']), + self::offer('workflow', 'second-packet'), self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame('empty', self::poll($client, 'workflow')['poll_status']); + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[3]['body']); + self::assertSame('second-packet', $transport->requests[4]['body']['task_offer_token']); + } + + public function testRegistrationCannotDowngradeAnUnresolvedAcceptance(): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Gateway timeout.', 504), ['registered' => true], self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { self::poll($client, 'workflow'); self::fail('Expected an uncertain acceptance.'); } + catch (TaskDeliveryUncertain) {} + try { $client->registerWorker('worker', 'orders', [], []); self::fail('Expected a rejected downgrade.'); } + catch (ServerException $error) { self::assertSame('invalid_task_delivery_capability', $error->reason); } + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + self::assertSame($transport->requests[2], $transport->requests[4]); + } + + public function testReplacementRegistrationDiscardsOnlyTheOldIncarnationsPacket(): void + { + $replacement = self::registration(); + $replacement['registration_token'] = str_repeat('b', 32); + $transport = new FakeTransport([self::registration(), self::offer('activity'), + new TransportException('Gateway timeout.', 504), $replacement, self::offer('activity', 'replacement-packet'), self::claimed('activity')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { self::poll($client, 'activity'); self::fail('Expected an uncertain acceptance.'); } + catch (TaskDeliveryUncertain) {} + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed('activity'), self::poll($client, 'activity')); + self::assertSame(str_repeat('b', 32), $transport->requests[4]['body']['registration_token']); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[4]['body']); + self::assertSame('replacement-packet', $transport->requests[5]['body']['task_offer_token']); + } + + #[DataProvider('legacyRegistrations')] + public function testLegacyDeliveryIsChosenOnlyAtRegistration(array $registration, string $protocol, ?array $capabilities): void + { + $transport = new FakeTransport([$registration, ['task' => null]]); + $client = new Client('https://server.example', transport: $transport, workerProtocolVersion: $protocol); + if ($capabilities === null) { + $client->registerWorker('worker', 'orders', [], []); + } else { + $client->registerWorker('worker', 'orders', [], [], $capabilities); + } + self::poll($client, 'workflow'); + self::assertStringEndsWith('/workflow-tasks/poll', $transport->requests[1]['uri']); + self::assertArrayNotHasKey(TaskDelivery::HEADER, $transport->requests[1]['headers']); + if ($protocol === '1.19') { + self::assertNotContains(TaskDelivery::CAPABILITY, $transport->requests[0]['body']['capabilities']); + } + } + + public static function legacyRegistrations(): iterable + { + yield 'no server feature' => [['registered' => true], '1.20', null]; + yield 'explicit worker opt-out' => [self::registration(), '1.20', ['query_tasks']]; + yield 'older protocol' => [self::registration(), '1.19', null]; + } + + public function testAContradictoryOfferCannotReachPayloadOrExecutionObservers(): void + { + $contradictory = self::claimed('workflow') + ['task_offer' => self::offer('workflow')['task_offer']]; + $transport = new FakeTransport([self::registration(), $contradictory]); + $claims = 0; + $client = (new Client('https://server.example', transport: $transport))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$claims): ?\Closure { + if ($response['task'] ?? null) { ++$claims; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'workflow'); + self::fail('A contradictory offer cannot authorize a task.'); + } catch (ServerException $error) { + self::assertSame('invalid_task_delivery_response', $error->reason); + } + self::assertSame(0, $claims); + self::assertCount(2, $transport->requests); + } + + public function testTheManagedWorkerRetainsItsLogicalPollAfterAnUncertainAcceptance(): void + { + $now = 0.0; + $retries = []; + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Not found.', 404), + ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']]); + $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders', workerId: 'worker', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + transientPollRetryObserver: static function (string $kind, int $attempt, float $delay, ServerException $error) use (&$retries): void { + $retries[] = [$kind, $attempt, $error::class]; + }); + $worker->run(4); + self::assertSame([['workflow', 1, TaskDeliveryUncertain::class]], $retries); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertContains(TaskDelivery::CAPABILITY, $transport->requests[0]['body']['capabilities']); + self::assertCount(4, $transport->requests); + } + + public function testOfferTransportUsesTheSameProgressAndLogicalPathForAllKinds(): void + { + foreach (['workflow', 'activity', 'query', 'update-validation'] as $kind) { + $path = '/worker/'.$kind.'-tasks/offers'; + self::assertSame($kind, PollTransport::taskKind('POST', $path)); + self::assertSame('/worker/'.$kind.'-tasks/poll', PollTransport::logicalPath($path)); + self::assertNull(PollTransport::taskKind('GET', $path)); + } + } + + public static function kinds(): array { return [['workflow'], ['activity'], ['query']]; } + + private static function poll(Client $client, string $kind): array + { + return match ($kind) { + 'workflow' => $client->pollWorkflowTaskResponse('worker', 'orders', 4, 'fixed-poll'), + 'activity' => $client->pollActivityTaskResponse('worker', 'orders', 4, 'fixed-poll'), + 'query' => $client->pollQueryTaskResponse('worker', 'orders', 4, 'fixed-poll'), + }; + } + + private static function offer(string $kind, string $token = 'opaque-first-packet'): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'task', + 'token' => $token, 'expires_at_ms' => 1000, + ]]; + } + + private static function claimed(string $kind): array + { + return ['task' => [($kind === 'query' ? 'query_task_id' : 'task_id') => 'task', 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } + + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'heartbeat_interval_seconds' => 10, + 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', + 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ], 'worker_deregistration_fencing' => ['schema' => 'durable-workflow.v2.worker-deregistration.v1', + 'supported' => true, 'endpoint' => '/worker/registrations/{workerId}/deregister', 'receipt_retention_seconds' => 600]]]; + } +} From 3ab9d59d02f2c81f84a47ccf73a87fd58d12857f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:26:57 +0000 Subject: [PATCH 02/12] Honor fenced terminal worker responses during offer acceptance --- src/Worker/TaskDelivery.php | 9 +++++---- tests/WorkerTaskDeliveryTest.php | 12 ++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index 9ab67953..590fffb3 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -95,10 +95,10 @@ public function poll(string $path, array $body, Closure $send): ?array if ($this->definitiveRefusal($exception)) { $this->resolved($key, $selected); $response = $exception->details ?? []; - if ($exception->reason === 'worker_registration_lost_authority') { - $this->forget($workerId); - } else { + if ($exception->reason === 'task_offer_expired') { $response['poll_status'] = 'empty'; + } else { + $this->forget($workerId); } return $response; @@ -167,7 +167,8 @@ private function definitiveRefusal(ServerException $exception): bool return $exception->status === 409 && $response !== null && !array_is_list($response) && array_key_exists('task', $response) && $response['task'] === null && ($response['reason'] ?? null) === $exception->reason - && in_array($exception->reason, ['task_offer_expired', 'worker_registration_lost_authority'], true); + && (in_array($exception->reason, ['task_offer_expired', 'worker_registration_lost_authority'], true) + || PollResponse::isTerminal($response)); } private function resolved(string $key, ?TaskDeliveryAttempt $selected): void diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php index 66a81788..5ab31ba0 100644 --- a/tests/WorkerTaskDeliveryTest.php +++ b/tests/WorkerTaskDeliveryTest.php @@ -128,6 +128,18 @@ public function testRegistrationCannotDowngradeAnUnresolvedAcceptance(): void self::assertSame($transport->requests[2], $transport->requests[4]); } + public function testAnAuthoritativeWorkerStopIsNotRetriedAsAnUncertainAcceptance(): void + { + $terminal = ['task' => null, 'poll_status' => 'draining', 'reason' => 'worker_draining']; + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Worker is draining.', 409, $terminal)]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame($terminal, self::poll($client, 'workflow')); + self::assertTrue(PollResponse::isTerminal($terminal)); + self::assertCount(3, $transport->requests); + } + public function testReplacementRegistrationDiscardsOnlyTheOldIncarnationsPacket(): void { $replacement = self::registration(); From 13700088131c74a00fb3b656eb5d2c1c8c543d5a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:34:27 +0000 Subject: [PATCH 03/12] Separate delivery refusal reconciliation from uncertain retry classification --- src/Worker/TaskDelivery.php | 57 +++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index 590fffb3..22287169 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -88,36 +88,18 @@ public function poll(string $path, array $body, Closure $send): ?array if ($response['task'] === null && !PollResponse::isTerminal($response)) { throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); } - $this->resolved($key, $selected); - - return $response; } catch (ServerException $exception) { - if ($this->definitiveRefusal($exception)) { - $this->resolved($key, $selected); - $response = $exception->details ?? []; - if ($exception->reason === 'task_offer_expired') { - $response['poll_status'] = 'empty'; - } else { - $this->forget($workerId); + $response = $this->refusedResponse($exception, $workerId); + if ($response === null) { + if ($this->isUncertain($exception, $selected)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); } - - return $response; - } - if ($exception instanceof TaskDeliveryUncertain) { throw $exception; } - if (($selected !== null && ($exception->status >= 500 || $exception->status === 409 || $exception->status === 429 - || $exception->reason === 'invalid_task_delivery_response' - || in_array($exception->status, [0, 404, 405], true))) - || ($selected === null && in_array($exception->status, [404, 405], true))) { - if (PollResponse::isTransientPayloadFailure($exception)) { - throw $exception; - } - throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); - } - - throw $exception; } + $this->resolved($key, $selected); + + return $response; } /** @@ -171,6 +153,31 @@ private function definitiveRefusal(ServerException $exception): bool || PollResponse::isTerminal($response)); } + /** @return array|null */ + private function refusedResponse(ServerException $exception, string $workerId): ?array + { + if (!$this->definitiveRefusal($exception)) { + return null; + } + $response = $exception->details ?? []; + if ($exception->reason === 'task_offer_expired') { + $response['poll_status'] = 'empty'; + } else { + $this->forget($workerId); + } + + return $response; + } + + private function isUncertain(ServerException $exception, ?TaskDeliveryAttempt $selected): bool + { + return !$exception instanceof TaskDeliveryUncertain && !PollResponse::isTransientPayloadFailure($exception) + && (($selected !== null && ($exception->status >= 500 || $exception->status === 409 || $exception->status === 429 + || $exception->reason === 'invalid_task_delivery_response' + || in_array($exception->status, [0, 404, 405], true))) + || ($selected === null && in_array($exception->status, [404, 405], true))); + } + private function resolved(string $key, ?TaskDeliveryAttempt $selected): void { if (($this->pending[$key] ?? null) === $selected) { From 704e2a4981664c86f451507882d80090ddac1b5d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sun, 1 Nov 2026 21:52:23 +0000 Subject: [PATCH 04/12] Give each logical worker poll one acceptance owner --- src/Worker/TaskDelivery.php | 60 +++++++++------- tests/WorkerTaskDeliveryTest.php | 114 +++++++++++++++++++++++++++++-- 2 files changed, 145 insertions(+), 29 deletions(-) diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index 22287169..c4245855 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -21,6 +21,8 @@ final class TaskDelivery private array $registrations = []; /** @var array */ private array $pending = []; + /** @var array */ + private array $accepting = []; /** * @param list $capabilities @@ -70,36 +72,44 @@ public function poll(string $path, array $body, Closure $send): ?array return null; } $key = hash('sha256', json_encode([$path, $workerId, $body['task_queue'], $body['poll_request_id']], JSON_THROW_ON_ERROR)); - $selected = $this->pending[$key] ?? null; + if (isset($this->accepting[$key])) { + return ['task' => null]; + } + // Hold ownership across both requests, including a suspended offer read. + $this->accepting[$key] = true; try { - if ($selected === null) { - $wirePath = substr($path, 0, -5).'/offers'; - $body['registration_token'] = $registration['token']; - $response = $send($wirePath, $body, null); - if (!array_key_exists('task_offer', $response)) { - return $response; + $selected = $this->pending[$key] ?? null; + try { + if ($selected === null) { + $wirePath = substr($path, 0, -5).'/offers'; + $body['registration_token'] = $registration['token']; + $response = $send($wirePath, $body, null); + if (!array_key_exists('task_offer', $response)) { + return $response; + } + $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, + (string) PollTransport::taskKind('POST', $path)); + $selected = $this->pending[$key] ??= $candidate; } - $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, - (string) PollTransport::taskKind('POST', $path)); - // Concurrent callers for one logical poll select only its first packet. - $selected = $this->pending[$key] ??= $candidate; - } - $response = $send($selected->path, $selected->requestBody(), $selected); - if ($response['task'] === null && !PollResponse::isTerminal($response)) { - throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); - } - } catch (ServerException $exception) { - $response = $this->refusedResponse($exception, $workerId); - if ($response === null) { - if ($this->isUncertain($exception, $selected)) { - throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); + $response = $send($selected->path, $selected->requestBody(), $selected); + if ($response['task'] === null && !PollResponse::isTerminal($response)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); + } + } catch (ServerException $exception) { + $response = $this->refusedResponse($exception, $workerId); + if ($response === null) { + if ($this->isUncertain($exception, $selected)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); + } + throw $exception; } - throw $exception; } - } - $this->resolved($key, $selected); + $this->resolved($key, $selected); - return $response; + return $response; + } finally { + unset($this->accepting[$key]); + } } /** diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php index 5ab31ba0..51899617 100644 --- a/tests/WorkerTaskDeliveryTest.php +++ b/tests/WorkerTaskDeliveryTest.php @@ -86,6 +86,112 @@ public static function uncertainOutcomes(): iterable yield 'wrong owner' => [['task' => ['task_id' => 'task', 'lease_owner' => 'other']]]; } + #[DataProvider('overlappingPolls')] + public function testOverlappingPollsHaveOnlyOneAcceptanceOwner(string $kind, bool $clone, string $pauseAt): void + { + $offers = $acceptances = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, $pauseAt, &$offers, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (isset($body['task_offer_token'])) { + if (++$acceptances === 1 && $pauseAt === 'accept') { \Fiber::suspend('accept'); } + return self::claimed($kind); + } + if (++$offers === 1 && $pauseAt === 'offer') { \Fiber::suspend('offer'); } + return self::offer($kind); + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static fn (): array => self::poll($client, $kind)); + self::assertSame($pauseAt, $owner->start()); + $contender = new \Fiber(static fn (): array => self::poll($other, $kind)); + $contender->start(); + self::assertTrue($contender->isTerminated()); + $owner->resume(); + self::assertSame(self::claimed($kind), $owner->getReturn()); + self::assertSame(['task' => null], $contender->getReturn()); + self::assertSame(1, $offers); + self::assertSame(1, $acceptances); + } + + #[DataProvider('overlappingClients')] + public function testUncertainAcceptanceOwnerReleasesTheGateWithoutReplacingItsPacket(string $kind, bool $clone): void + { + $offers = $acceptances = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, &$offers, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (!isset($body['task_offer_token'])) { return self::offer($kind, 'packet-'.++$offers); } + if (++$acceptances === 1) { + \Fiber::suspend('accept'); + throw new TransportException('Gateway timeout.', 504); + } + return self::claimed($kind); + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static function () use ($client, $kind): ?TaskDeliveryUncertain { + try { self::poll($client, $kind); } + catch (TaskDeliveryUncertain $error) { return $error; } + return null; + }); + self::assertSame('accept', $owner->start()); + $contender = new \Fiber(static fn (): array => self::poll($other, $kind)); + $contender->start(); + $owner->resume(); + self::assertInstanceOf(TaskDeliveryUncertain::class, $owner->getReturn()); + self::assertSame(['task' => null], $contender->getReturn()); + self::assertSame(self::claimed($kind), self::poll($other, $kind)); + self::assertSame(1, $offers); + self::assertSame(2, $acceptances); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + public static function overlappingClients(): iterable + { + foreach (['workflow', 'activity', 'query'] as $kind) { + yield $kind.' same client' => [$kind, false]; + yield $kind.' clone' => [$kind, true]; + } + } + + #[DataProvider('overlappingClients')] + public function testAnAcceptanceOwnerDoesNotBlockAnotherLogicalPoll(string $kind, bool $clone): void + { + $acceptances = 0; + $idField = $kind === 'query' ? 'query_task_id' : 'task_id'; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, $idField, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (!isset($body['task_offer_token'])) { + $response = self::offer($kind); + $response['task_offer']['task_id'] = $body['poll_request_id']; + return $response; + } + if (++$acceptances === 1) { \Fiber::suspend('accept'); } + $response = self::claimed($kind); + $response['task'][$idField] = $body['poll_request_id']; + return $response; + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static fn (): array => self::poll($client, $kind)); + self::assertSame('accept', $owner->start()); + self::assertSame('other-poll', self::poll($other, $kind, 'other-poll')['task'][$idField]); + $owner->resume(); + self::assertSame('fixed-poll', $owner->getReturn()['task'][$idField]); + self::assertSame(2, $acceptances); + } + + public static function overlappingPolls(): iterable + { + foreach (self::overlappingClients() as $name => $arguments) { + foreach (['offer', 'accept'] as $phase) { + yield $name.' during '.$phase => [...$arguments, $phase]; + } + } + } + public function testAnOlderNodeCannotDowngradeTheNegotiatedEndpoint(): void { $transport = new FakeTransport([self::registration(), new TransportException('Not found.', 404), ['task' => null]]); @@ -235,12 +341,12 @@ public function testOfferTransportUsesTheSameProgressAndLogicalPathForAllKinds() public static function kinds(): array { return [['workflow'], ['activity'], ['query']]; } - private static function poll(Client $client, string $kind): array + private static function poll(Client $client, string $kind, string $pollId = 'fixed-poll'): array { return match ($kind) { - 'workflow' => $client->pollWorkflowTaskResponse('worker', 'orders', 4, 'fixed-poll'), - 'activity' => $client->pollActivityTaskResponse('worker', 'orders', 4, 'fixed-poll'), - 'query' => $client->pollQueryTaskResponse('worker', 'orders', 4, 'fixed-poll'), + 'workflow' => $client->pollWorkflowTaskResponse('worker', 'orders', 4, $pollId), + 'activity' => $client->pollActivityTaskResponse('worker', 'orders', 4, $pollId), + 'query' => $client->pollQueryTaskResponse('worker', 'orders', 4, $pollId), }; } From 82ba543131fb6a75898dff67b7a3e30069166016 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:43:06 +0000 Subject: [PATCH 05/12] Bind task delivery kinds to the original logical poll --- src/Worker/TaskDelivery.php | 56 ++- src/Worker/TaskDeliveryAttempt.php | 21 +- tests/WorkerTaskDeliveryKindsTest.php | 333 ++++++++++++++++ tests/WorkerTaskDeliveryTest.php | 37 ++ .../fixtures/task-delivery-contract/README.md | 19 + .../offer-accept.v1.json | 355 ++++++++++++++++++ 6 files changed, 807 insertions(+), 14 deletions(-) create mode 100644 tests/WorkerTaskDeliveryKindsTest.php create mode 100644 tests/fixtures/task-delivery-contract/README.md create mode 100644 tests/fixtures/task-delivery-contract/offer-accept.v1.json diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index c4245855..272adcc9 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -8,6 +8,7 @@ use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Exception\TaskDeliveryUncertain; use DurableWorkflow\Transport\PollTransport; +use InvalidArgumentException; /** Per-Client registration and unresolved delivery state, shared by its clones. */ final class TaskDelivery @@ -21,7 +22,7 @@ final class TaskDelivery private array $registrations = []; /** @var array */ private array $pending = []; - /** @var array */ + /** @var array> */ private array $accepting = []; /** @@ -71,14 +72,22 @@ public function poll(string $path, array $body, Closure $send): ?array if ($registration === null || $registration['queue'] !== ($body['task_queue'] ?? null)) { return null; } + $allowedKinds = self::requestedKinds($path, $body); + if (array_key_exists('task_kinds', $body)) { + $body['task_kinds'] = $allowedKinds; + } $key = hash('sha256', json_encode([$path, $workerId, $body['task_queue'], $body['poll_request_id']], JSON_THROW_ON_ERROR)); + $selected = $this->pending[$key] ?? null; + if (($selected !== null && !$selected->isForKinds($allowedKinds)) + || (isset($this->accepting[$key]) && $this->accepting[$key] !== $allowedKinds)) { + throw new InvalidArgumentException('A logical task poll cannot change its requested task kinds.'); + } if (isset($this->accepting[$key])) { return ['task' => null]; } // Hold ownership across both requests, including a suspended offer read. - $this->accepting[$key] = true; + $this->accepting[$key] = $allowedKinds; try { - $selected = $this->pending[$key] ?? null; try { if ($selected === null) { $wirePath = substr($path, 0, -5).'/offers'; @@ -87,8 +96,7 @@ public function poll(string $path, array $body, Closure $send): ?array if (!array_key_exists('task_offer', $response)) { return $response; } - $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, - (string) PollTransport::taskKind('POST', $path)); + $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, $allowedKinds); $selected = $this->pending[$key] ??= $candidate; } $response = $send($selected->path, $selected->requestBody(), $selected); @@ -120,22 +128,50 @@ public function poll(string $path, array $body, Closure $send): ?array public static function assertResponse(array $response, array $body, string $path, ?TaskDeliveryAttempt $selected): void { $task = $response['task'] ?? null; - $kind = PollTransport::taskKind('POST', $path); + $kind = $selected?->taskKind; $idField = match ($kind) { - 'query' => 'query_task_id', 'update-validation' => 'update_validation_task_id', default => 'task_id', + 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id', }; if (!array_key_exists('task', $response) || (($selected !== null || $task !== null) && array_key_exists('task_offer', $response)) || (($response['poll_status'] ?? null) === 'offered' && !array_key_exists('task_offer', $response)) - || ($task !== null && (!is_array($task) || array_is_list($task) + || ($task !== null && ($selected === null || !is_array($task) || array_is_list($task) || !is_string($task[$idField] ?? null) || $task[$idField] === '' || ($task['lease_owner'] ?? null) !== $body['worker_id'] - || ($selected !== null && $task[$idField] !== $selected->taskId) - || (isset($task['task_kind']) && $task['task_kind'] !== str_replace('-', '_', (string) $kind))))) { + || $task[$idField] !== $selected->taskId + || ($task['task_kind'] ?? null) !== $kind + || !$selected->isForKinds(self::requestedKinds($path, $body))))) { throw new ServerException('The task-delivery response is invalid.', 200, 'invalid_task_delivery_response'); } } + /** + * @param array $body + * @return list + */ + private static function requestedKinds(string $path, array $body): array + { + $kind = str_replace('-', '_', (string) PollTransport::taskKind('POST', $path)); + if (!in_array($kind, ['workflow', 'activity', 'query', 'update_validation'], true)) { + throw new InvalidArgumentException('Task delivery requires a supported poll endpoint.'); + } + $permitted = $kind === 'workflow' ? ['workflow', 'update_validation'] : [$kind]; + $requested = array_key_exists('task_kinds', $body) ? $body['task_kinds'] : [$kind]; + if (!is_array($requested) || !array_is_list($requested) || $requested === []) { + throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); + } + foreach ($requested as $value) { + if (!is_string($value) || !in_array($value, $permitted, true)) { + throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); + } + } + /** @var list $requested */ + $requested = array_values(array_unique($requested)); + sort($requested, SORT_STRING); + + return $requested; + } + public function forget(string $workerId): void { unset($this->registrations[$workerId]); diff --git a/src/Worker/TaskDeliveryAttempt.php b/src/Worker/TaskDeliveryAttempt.php index 64cff2ca..636458b4 100644 --- a/src/Worker/TaskDeliveryAttempt.php +++ b/src/Worker/TaskDeliveryAttempt.php @@ -9,11 +9,16 @@ /** The first selected packet remains immutable through uncertain replies. */ final class TaskDeliveryAttempt { - /** @param array $body */ + /** + * @param list $allowedKinds + * @param array $body + */ private function __construct( public readonly string $workerId, public readonly string $path, public readonly string $taskId, + public readonly string $taskKind, + private readonly array $allowedKinds, private readonly array $body, ) { } @@ -21,24 +26,32 @@ private function __construct( /** * @param array $body * @param array $response + * @param list $allowedKinds */ - public static function fromResponse(string $path, array $body, array $response, string $kind): self + public static function fromResponse(string $path, array $body, array $response, array $allowedKinds): self { $offer = $response['task_offer'] ?? null; if (!array_key_exists('task', $response) || $response['task'] !== null || ($response['poll_status'] ?? null) !== 'offered' || !is_array($offer) || array_is_list($offer) || ($offer['schema'] ?? null) !== TaskDelivery::SCHEMA - || ($offer['task_kind'] ?? null) !== str_replace('-', '_', $kind) + || !is_string($offer['task_kind'] ?? null) + || !in_array($offer['task_kind'], $allowedKinds, true) || !is_string($offer['task_id'] ?? null) || $offer['task_id'] === '' || !is_string($offer['token'] ?? null) || $offer['token'] === '' || strlen($offer['token']) > 8192 || !is_int($offer['expires_at_ms'] ?? null) || $offer['expires_at_ms'] < 1) { throw new ServerException('The server returned an invalid task offer.', 200, 'invalid_task_delivery_offer'); } - return new self((string) $body['worker_id'], $path, $offer['task_id'], + return new self((string) $body['worker_id'], $path, $offer['task_id'], $offer['task_kind'], $allowedKinds, array_replace($body, ['task_offer_token' => $offer['token'], 'timeout_seconds' => 0])); } + /** @param list $allowedKinds */ + public function isForKinds(array $allowedKinds): bool + { + return $this->allowedKinds === $allowedKinds; + } + /** @return array */ public function requestBody(): array { diff --git a/tests/WorkerTaskDeliveryKindsTest.php b/tests/WorkerTaskDeliveryKindsTest.php new file mode 100644 index 00000000..16c0a2d6 --- /dev/null +++ b/tests/WorkerTaskDeliveryKindsTest.php @@ -0,0 +1,333 @@ + str_starts_with($id, 'kind-')))); + } + + /** @param list|null $kinds */ + #[DataProvider('validKinds')] + public function testOnlyRequestedKindsCanProduceTheirSelectedReceipt(?array $kinds, string $selected, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), self::offer($selected), self::claimed($selected)]); + $client = self::client($transport); + [$result, $error] = self::capture(static fn (): array => self::poll($client, $kinds, $endpoint)); + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame(self::claimed($selected), $result); + self::assertCount(3, $transport->requests); + $offer = $transport->requests[1]; + $accept = $transport->requests[2]; + self::assertStringEndsWith('/'.$endpoint.'-tasks/offers', $offer['uri']); + self::assertSame($offer['uri'], $accept['uri']); + $originalBody = $offer['body']; + self::assertIsArray($originalBody); + self::assertSame(array_replace($originalBody, ['timeout_seconds' => 0, 'task_offer_token' => 'selected-packet']), $accept['body']); + if ($kinds !== null) { + self::assertSame(in_array('workflow', $kinds, true) && in_array('update_validation', $kinds, true) ? self::BOTH : $kinds, + $offer['body']['task_kinds']); + } + } + + /** @return iterable|null, string, string}> */ + public static function validKinds(): iterable + { + $data = json_decode((string) file_get_contents(self::FIXTURE), true, flags: JSON_THROW_ON_ERROR); + foreach ($data['cases'] as $case) { + if (in_array($case['id'], ['kind-workflow', 'kind-validator', 'kind-mixed-order'], true)) { + foreach ($case['request_kind_orders'] ?? [$case['requested_kinds']] as $index => $kinds) { + yield $case['id'].' / '.$index => [$kinds, $case['offered_kind'], $case['endpoint']]; + } + } + } + yield 'mixed workflow / order 1' => [self::BOTH, 'workflow', 'workflow']; + yield 'mixed workflow / order 2' => [['workflow', 'update_validation'], 'workflow', 'workflow']; + yield 'normalized duplicate kinds' => [['workflow', 'update_validation', 'workflow'], 'update_validation', 'workflow']; + yield 'default workflow set' => [null, 'workflow', 'workflow']; + foreach (['activity', 'query', 'update-validation'] as $endpoint) { + $kind = str_replace('-', '_', $endpoint); + yield 'single-kind '.$endpoint => [[$kind], $kind, $endpoint]; + } + } + + /** @param list $kinds */ + #[DataProvider('unrequestedKinds')] + public function testAnUnrequestedOfferSendsNoAcceptance(array $kinds, string $offered, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), self::offer($offered), self::claimed($offered)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, $kinds, $endpoint)); + self::assertInstanceOf(ServerException::class, $error); + self::assertSame('invalid_task_delivery_offer', $error->reason); + self::assertCount(2, $transport->requests); + } + + /** @return iterable, string, string}> */ + public static function unrequestedKinds(): iterable + { + yield 'kind-unrequested' => [['workflow'], 'update_validation', 'workflow']; + yield 'validator-only cannot select workflow' => [['update_validation'], 'workflow', 'workflow']; + foreach (['activity', 'query', 'update-validation'] as $endpoint) { + yield $endpoint => [[str_replace('-', '_', $endpoint)], 'workflow', $endpoint]; + } + } + + #[DataProvider('receiptMutations')] + public function testSubstitutedOrMissingReceiptIdentityCannotReleaseThePacket(string $kind, string $mutation): void + { + $bad = self::claimed($kind); + $field = $kind === 'update_validation' ? 'update_validation_task_id' : 'task_id'; + if ($mutation === 'kind-substitution') { + $bad['task']['task_kind'] = $kind === 'workflow' ? 'update_validation' : 'workflow'; + $bad['task']['task_id'] = $bad['task']['update_validation_task_id'] = 'selected-task'; + } elseif ($mutation === 'missing-kind') { + unset($bad['task']['task_kind']); + } elseif ($mutation === 'null-kind') { + $bad['task']['task_kind'] = null; + } elseif ($mutation === 'task-substitution') { + $bad['task'][$field] = 'other-task'; + } else { + unset($bad['task'][$field]); + $bad['task'][$field === 'task_id' ? 'update_validation_task_id' : 'task_id'] = 'selected-task'; + } + $transport = new FakeTransport([self::registration(), self::offer($kind), $bad, self::claimed($kind)]); + $observed = 0; + $client = self::client($transport)->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(0, $observed); + self::assertSame(self::claimed($kind), self::poll(clone $client, ['workflow', 'update_validation'])); + self::assertSame(1, $observed); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + /** @return iterable */ + public static function receiptMutations(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (['kind-substitution', 'missing-kind', 'null-kind', 'task-substitution', 'wrong-id-field'] as $mutation) { + yield $kind.' / '.$mutation => [$kind, $mutation]; + } + } + } + + #[DataProvider('selectedKinds')] + public function testChangedKindsCannotReplaceAnUncertainOriginalRequest(string $kind): void + { + $transport = new FakeTransport([self::registration(), self::offer($kind), new TransportException('Timeout.', 504), self::claimed($kind)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + $clone = clone $client; + [, $error] = self::capture(static fn (): array => self::poll($clone, ['workflow'])); + self::assertInstanceOf(InvalidArgumentException::class, $error); + self::assertCount(3, $transport->requests); + self::assertSame(self::claimed($kind), self::poll($clone, ['workflow', 'update_validation', 'workflow'])); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + /** @return array */ + public static function selectedKinds(): array { return [['workflow'], ['update_validation']]; } + + #[DataProvider('concurrentKinds')] + public function testKindSetBindingSpansOfferAndAcceptanceOwnership(string $kind, string $phase, bool $clone, bool $changed): void + { + $offers = $accepts = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ( + $kind, $phase, &$offers, &$accepts, + ): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (isset($body['task_offer_token'])) { + ++$accepts; + if ($phase === 'accept') { \Fiber::suspend('accept'); } + return self::claimed($kind); + } + ++$offers; + if ($phase === 'offer') { \Fiber::suspend('offer'); } + return self::offer($kind); + }); + $client = self::client($transport); + $owner = new \Fiber(static fn (): array => self::poll($client, self::BOTH)); + try { $paused = $owner->start(); } + catch (ServerException $error) { self::fail('The valid owner was refused before its pause: '.$error->reason); } + self::assertSame($phase, $paused); + $other = $clone ? clone $client : $client; + [$result, $error] = self::capture(static fn (): array => self::poll($other, $changed ? ['workflow'] : ['workflow', 'update_validation'])); + try { $owner->resume(); } + catch (ServerException $error) { self::fail('The valid owner was refused after its pause: '.$error->reason); } + self::assertSame(self::claimed($kind), $owner->getReturn()); + if ($changed) { + self::assertInstanceOf(InvalidArgumentException::class, $error); + } else { + self::assertNull($error); + self::assertSame(['task' => null], $result); + } + self::assertSame(1, $offers); + self::assertSame(1, $accepts); + self::assertCount(3, $transport->requests); + } + + /** @return iterable */ + public static function concurrentKinds(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (['offer', 'accept'] as $phase) { + foreach ([false, true] as $clone) { + foreach ([false, true] as $changed) { + yield $kind.' / '.$phase.' / '.(int) $clone.' / '.(int) $changed => [$kind, $phase, $clone, $changed]; + } + } + } + } + } + + #[DataProvider('invalidKindSets')] + public function testMalformedKindSetsRefuseBeforeAnOffer(mixed $kinds, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), ['task' => null]]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::rawPoll($client, ['task_kinds' => $kinds], $endpoint)); + self::assertInstanceOf(InvalidArgumentException::class, $error); + self::assertCount(1, $transport->requests); + } + + /** @return iterable */ + public static function invalidKindSets(): iterable + { + foreach ([null, '', [], ['kind' => 'workflow'], [1], ['activity'], ['update-validation'], ['workflow', 'foreign']] as $index => $kinds) { + yield (string) $index => [$kinds, 'workflow']; + } + foreach (['activity', 'query', 'update-validation', 'unknown'] as $endpoint) { + yield $endpoint => [['workflow'], $endpoint]; + } + } + + /** @param TransportException|array|null $outcome */ + #[DataProvider('uncertainKinds')] + public function testUncertainMixedRepliesRetainTheExactOriginalPacket(string $kind, TransportException|array|null $outcome): void + { + $transport = new FakeTransport([self::registration(), self::offer($kind), $outcome, self::claimed($kind)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(self::claimed($kind), self::poll(clone $client, ['workflow', 'update_validation'])); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertStringEndsWith('/workflow-tasks/offers', $transport->requests[3]['uri']); + } + + /** @return iterable|null}> */ + public static function uncertainKinds(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (WorkerTaskDeliveryTest::uncertainOutcomes() as $name => [$outcome]) { + yield $kind.' / '.$name => [$kind, $outcome]; + } + } + } + + public function testAnExecutableTaskWithoutASelectedOfferCannotBypassKindBinding(): void + { + $transport = new FakeTransport([self::registration(), self::claimed('workflow')]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, ['workflow'])); + self::assertInstanceOf(ServerException::class, $error); + self::assertSame('invalid_task_delivery_response', $error->reason); + self::assertCount(2, $transport->requests); + } + + /** + * @param list|null $kinds + * @return array + */ + private static function poll(Client $client, ?array $kinds, string $endpoint = 'workflow'): array + { + return self::rawPoll($client, $kinds === null ? [] : ['task_kinds' => $kinds], $endpoint); + } + + /** + * @param array $extra + * @return array + */ + private static function rawPoll(Client $client, array $extra, string $endpoint = 'workflow'): array + { + // Exercise the same logical-request boundary used by the public poll + // methods, without adding update-validator authoring to this source slice. + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => 'orders', 'poll_request_id' => 'kind-poll', 'timeout_seconds' => 4, + ...$extra, + ]); + } + + /** @return array{array|null, ServerException|InvalidArgumentException|null} */ + private static function capture(Closure $operation): array + { + try { return [$operation(), null]; } + catch (ServerException|InvalidArgumentException $error) { return [null, $error]; } + } + + private static function client(FakeTransport $transport): Client + { + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + return $client; + } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, + 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'selected-task', + 'token' => 'selected-packet', 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => 'selected-task', 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } +} diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php index 51899617..48075982 100644 --- a/tests/WorkerTaskDeliveryTest.php +++ b/tests/WorkerTaskDeliveryTest.php @@ -9,6 +9,7 @@ use DurableWorkflow\Exception\TaskDeliveryUncertain; use DurableWorkflow\Exception\TransportException; use DurableWorkflow\Tests\Support\FakeTransport; +use DurableWorkflow\Tests\Support\RecordingMetricsRecorder; use DurableWorkflow\Transport\PollTransport; use DurableWorkflow\Worker; use DurableWorkflow\Worker\PollResponse; @@ -18,6 +19,42 @@ final class WorkerTaskDeliveryTest extends TestCase { + #[DataProvider('metricReceipts')] + public function testMetricsKeepOneLogicalPollAndValidateBeforePayloadObservation(bool $invalidFirst): void + { + $first = self::claimed('workflow'); + if ($invalidFirst) { + unset($first['task']['task_kind']); + } + $transport = new FakeTransport([self::registration(), self::offer('workflow'), $first, self::claimed('workflow')]); + $recorder = new RecordingMetricsRecorder(); + $observed = 0; + $client = (new Client('https://server.example', transport: $transport, metrics: $recorder))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?\Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + if ($invalidFirst) { + try { self::poll($client, 'workflow'); self::fail('The selected receipt must carry its kind.'); } + catch (TaskDeliveryUncertain) {} + self::assertSame(0, $observed); + self::assertSame(self::claimed('workflow'), self::poll(clone $client, 'workflow')); + self::assertSame($transport->requests[2], $transport->requests[3]); + } else { + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + } + self::assertSame(1, $observed); + $polls = array_values(array_filter($recorder->events, static fn (array $event): bool => + $event['name'] === 'durable_workflow_client_requests' && $event['tags']['route'] === '/worker/workflow-tasks/poll')); + self::assertCount($invalidFirst ? 2 : 1, $polls); + self::assertCount($invalidFirst ? 4 : 3, $transport->requests); + self::assertStringNotContainsString('opaque-first-packet', json_encode($recorder->events, JSON_THROW_ON_ERROR)); + } + + /** @return array */ + public static function metricReceipts(): array { return [[false], [true]]; } + #[DataProvider('kinds')] public function testOnlyPositiveAcceptanceReturnsAnExecutableTask(string $kind): void { diff --git a/tests/fixtures/task-delivery-contract/README.md b/tests/fixtures/task-delivery-contract/README.md new file mode 100644 index 00000000..c261c0e8 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/README.md @@ -0,0 +1,19 @@ +# Task delivery source contract + +`offer-accept.v1.json` is byte-identical to Server commit +`955da4e870ef76e7df1bcd24beaae037df63a8f4`, path +`tests/Fixtures/contracts/task-delivery-offers/offer-accept.v1.json`: + +- Git blob: `92df9dd2243521a509bfc8b6ce61a7be16711060` +- SHA-256: `cac2bc6852a09db9526c623c3683772ebbba7d3f1d0b401fb92ae2026cd32b11` +- Size: 18,142 bytes; 49 scenario IDs. + +The source contract is reviewed in Server #1033 for the unimplemented native +#931 capability. Its status remains draft contract data, without an executor +or qualification result. `WorkerTaskDeliveryKindsTest` drives the real PHP +Client/delivery state through an in-memory transport for the five `kind-*` +rows only. Supplementary PHP controls cover malformed kind sets, immutable +selection, clones and uncertainty. They do not execute the other native +claim/receipt, seal/key, database, proxy, restart or capacity scenarios. Reading +or enumerating those rows supplies no execution result. Tokens are synthetic +opaque test text, not sealed native packets or real credentials. diff --git a/tests/fixtures/task-delivery-contract/offer-accept.v1.json b/tests/fixtures/task-delivery-contract/offer-accept.v1.json new file mode 100644 index 00000000..92df9dd2 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/offer-accept.v1.json @@ -0,0 +1,355 @@ +{ + "schema": "durable-workflow.task-delivery-scenarios.v1", + "status": "draft contract data; no executor or qualification result", + "source_issue": "https://github.com/durable-workflow/server/issues/931", + "reference_sources": { + "retired_server_php_commit": "ed302326f0f73c383df88a918c8378755e0a56fe", + "held_sdk_php_commit": "034c0f4dc2fbbc1671fe9b1cdfd0fa36c2576c2e" + }, + "contract": { + "schema": "durable-workflow.task-delivery.v1", + "version": "offer-accept.v1", + "minimum_protocol_version": "1.20", + "worker_capability": "task_delivery_offers", + "header": "X-Durable-Workflow-Task-Delivery", + "endpoint_suffix": "/offers", + "acceptance_timeout_seconds": { + "default": 2, + "minimum": 1, + "maximum": 10, + "actual_integer_required": true + }, + "offer_token_maximum_bytes": 8192, + "unaccepted_offer_reserves_task": false, + "endpoints": { + "workflow": ["workflow", "update_validation"], + "activity": ["activity"], + "query": ["query"], + "update-validation": ["update_validation"] + } + }, + "executor_requirements": { + "inputs": "synthetic authorized registration, tasks, clocks and faults; no real credentials", + "packets": "seal synthetic authoritative contexts with the reviewed implementation; do not use placeholder tokens", + "database": "exercise actual claim/receipt transactions, not a scenario-only model", + "comparison": "compare task authority and attempts, start history and receipt before and after every phase", + "scope": "run on each source backend required by the owning PR; installed proxy/restart/capacity acceptance is separate", + "no_result_inference": "parsing or enumerating this data is not a passing execution" + }, + "security_source_direction": { + "review": "https://github.com/durable-workflow/server/issues/931#issuecomment-6646286980", + "status": "design direction accepted; implementation acceptance held", + "algorithm": "full-round-XChaCha20Poly1305", + "key_bytes": 32, + "nonce_bytes": 24, + "tag_bytes": 16, + "maximum_plaintext_bytes": 6000, + "decoded_frame_bytes": {"minimum": 40, "maximum": 6040}, + "maximum_frame_bytes_with_32_byte_key_id": 8092, + "requirements": ["exact-reviewed-locked-graph", "supported-zeroization-feature", "fallible-os-rng", "closed-canonical-parsers", "persisted-key-id-references", "enforceable-rotation-and-startup-preflight"], + "key_ring_proposal": { + "setting": "DW_TASK_DELIVERY_SEAL_KEYS", + "maximum_raw_utf8_bytes": 4096, + "absent": "unsupported", + "supplied_empty": "configuration-refusal", + "container_fields": ["schema", "keys"], + "schema": "durable-workflow.task-delivery-seal-keys.v1", + "minimum_entries": 1, + "maximum_entries": 4, + "entry_fields": ["id", "key_base64"], + "key_id_ascii_pattern": "[A-Za-z0-9_-]{1,32}", + "key_text_encoding": "RFC4648-standard-padded-canonical", + "key_text_ascii_bytes": 44, + "key_text_trailing_padding_bytes": 1, + "canonical_reencode_required": true, + "validate_all_original_key_text_before_any_decode_or_selection": true, + "validate_container_formatting_as_credential_text": false + } + }, + "cases": [ + { + "id": "offer-lost", + "kinds": ["workflow", "activity", "query", "update_validation"], + "actions": ["observe-ready-task", "drop-offer-response", "peer-claim"], + "expected": {"offer_started_attempts": 0, "offer_start_events": 0, "peer_claims_same_task": true} + }, + { + "id": "offer-peer-wins", + "actions": ["observe-ready-task", "peer-claims-task", "accept-original-offer"], + "expected": {"new_claims_from_accept": 0, "peer_authority_unchanged": true} + }, + { + "id": "offer-expired-unaccepted", + "actions": ["observe-ready-task", "advance-past-offer-expiry", "accept-original-offer", "start-new-logical-poll"], + "expected": {"refusal_status": 409, "reason": "task_offer_expired", "task": null, "old_accept_started_attempts": 0, "new_poll_eligible": true} + }, + { + "id": "accept-lost", + "actions": ["observe-ready-task", "accept-and-commit", "drop-accept-response", "retry-identical-packet"], + "expected": {"total_new_claims": 1, "same_task_owner_attempt": true, "new_start_events_on_retry": 0} + }, + { + "id": "accept-after-offer-expiry", + "actions": ["observe-ready-task", "accept-and-commit", "advance-past-offer-expiry-with-live-lease", "retry-identical-packet"], + "expected": {"replay_accepted_claim": true, "total_new_claims": 1, "new_start_events_on_retry": 0} + }, + { + "id": "accept-concurrent-same", + "actions": ["observe-ready-task", "accept-same-packet-concurrently-through-independent-connections"], + "expected": {"total_new_claims": 1, "all_delivered_claims_same_task_owner_attempt": true} + }, + { + "id": "accept-alternative-offers", + "actions": ["observe-two-eligible-tasks-under-same-logical-poll", "accept-alternative-packets-concurrently"], + "expected": {"maximum_committed_claims": 1, "unselected_task_claimable_by_peer": true, "no_other_task_substituted_for_selected_packet": true} + }, + { + "id": "accept-after-lease-renewal", + "actions": ["accept-and-commit", "renew-same-claim", "advance-past-original-lease-expiry-before-renewed-expiry", "retry-identical-packet"], + "expected": {"replay_current_live_claim": true, "same_task_owner_attempt": true, "new_start_events_on_retry": 0} + }, + { + "id": "accept-after-task-replacement", + "actions": ["accept-and-commit", "end-claim-and-replace-attempt", "retry-original-packet"], + "expected": {"old_claim_delivered": false, "new_claims_from_retry": 0, "replacement_authority_unchanged": true} + }, + { + "id": "registration-replaced", + "actions": ["observe-ready-task", "register-replacement-worker", "accept-old-packet"], + "expected": {"refusal_status": 409, "reason": "worker_registration_lost_authority", "task": null, "new_claims_from_accept": 0} + }, + { + "id": "same-second-replaced", + "actions": ["observe-ready-task", "register-same-worker-id-and-process-metadata-within-same-stored-second", "accept-old-packet"], + "expected": {"refusal_status": 409, "reason": "worker_registration_lost_authority", "task": null, "new_claims_from_accept": 0} + }, + { + "id": "routing-changed", + "actions": ["observe-ready-task", "change-registration-routing-fence", "accept-old-packet"], + "expected": {"new_claims_from_accept": 0, "old_packet_has_authority": false} + }, + { + "id": "cross-namespace", + "actions": ["observe-ready-task", "submit-packet-in-another-authorized-namespace"], + "expected": {"new_claims_from_accept": 0, "neither_namespace_task_authority_changed": true} + }, + { + "id": "changed-packet", + "actions": ["observe-ready-task", "change-one-sealed-byte", "accept-mutated-packet"], + "expected": {"new_claims_from_accept": 0, "seal_rejected_before_claim": true} + }, + { + "id": "kind-workflow", + "endpoint": "workflow", + "requested_kinds": ["workflow"], + "offered_kind": "workflow", + "expected": {"selected_kind_allowed": true, "accepted_kind_must_equal": "workflow"} + }, + { + "id": "kind-validator", + "endpoint": "workflow", + "requested_kinds": ["update_validation"], + "offered_kind": "update_validation", + "expected": {"selected_kind_allowed": true, "accepted_kind_must_equal": "update_validation"} + }, + { + "id": "kind-mixed-order", + "endpoint": "workflow", + "request_kind_orders": [["workflow", "update_validation"], ["update_validation", "workflow"]], + "offered_kind": "update_validation", + "expected": {"same_normalized_kind_set": true, "selected_kind_allowed": true, "accepted_kind_must_equal": "update_validation"} + }, + { + "id": "kind-unrequested", + "endpoint": "workflow", + "requested_kinds": ["workflow"], + "offered_kind": "update_validation", + "expected": {"selected_kind_allowed": false, "accept_requests_sent": 0} + }, + { + "id": "kind-accepted-substitution", + "endpoint": "workflow", + "requested_kinds": ["workflow", "update_validation"], + "selected_kind": "update_validation", + "accepted_kind": "workflow", + "expected": {"execution_authorized": false, "discard_uncertain_selected_packet": false} + }, + { + "id": "accept-db-precommit-fault", + "actions": ["observe-ready-task", "fault-after-claim-writes-before-commit", "inspect-authoritative-storage"], + "expected": {"new_claims_committed": 0, "new_attempts_committed": 0, "new_start_events_committed": 0, "accepted_receipts_committed": 0} + }, + { + "id": "accept-db-uncertain-commit", + "actions": ["observe-ready-task", "commit-acceptance", "lose-database-commit-response", "retry-identical-packet-through-new-connection"], + "expected": {"total_new_claims": 1, "same_task_owner_attempt": true, "legacy_poll_requests": 0} + }, + { + "id": "mixed-old-node", + "actions": ["accept-and-commit", "drop-accept-response", "route-identical-retry-to-old-node-404-or-405", "retry-identical-packet-on-capable-node"], + "expected": {"retain_selected_packet_after_old_node": true, "legacy_poll_requests": 0, "total_new_claims": 1} + }, + { + "id": "proxy-short-wait", + "scope": "separately admitted installed acceptance", + "actions": ["cut-offer-poll-before-configured-wait", "start-task-in-abandoned-poll-window", "observe-live-worker-delivery-and-warning-diagnostics"], + "expected": {"abandoned_task_leases": 0, "wait_capacity_starvation": false, "observed_cutoff_and_configured_wait_reported": true} + }, + { + "id": "restart-wait-capacity", + "scope": "separately admitted installed acceptance", + "actions": ["hold-wait-capacity", "abruptly-restart-server-process", "resume-api-with-same-durable-state", "admit-live-waiter"], + "expected": {"dead_process_owns_wait_slots": false, "task_authority_derived_from_capacity_hint": false} + }, + { + "id": "worker-replacement-capacity", + "scope": "separately admitted installed acceptance", + "actions": ["hold-old-worker-poll", "replace-worker-registration", "release-old-owner-wait-capacity", "poll-as-replacement"], + "expected": {"old_worker_can_claim": false, "replacement_eligible": true, "global_and_namespace_limits_preserved": true} + }, + { + "id": "other-replica-live-waiter", + "scope": "separately admitted installed acceptance", + "actions": ["hold-live-waiter-on-peer-replica", "clean-up-dead-local-owner"], + "expected": {"peer_live_waiter_preserved": true, "foreign_owner_slot_released": false, "global_and_namespace_limits_preserved": true} + }, + { + "id": "nonce-rng-failure", + "scope": "required source controls; not executed", + "actions": ["inject-fallible-os-rng-error", "request-offer"], + "expected": {"packets_emitted": 0, "typed_refusal": true} + }, + { + "id": "nonce-reuse-injection", + "scope": "required source controls; not executed", + "actions": ["inject-repeated-test-nonce", "exercise-seal-wrapper-countercontrol"], + "expected": {"reuse_exposes_refusal_or_failing_countercontrol": true, "production_caller_nonce_accepted": false} + }, + { + "id": "frame-canonical-encoding", + "scope": "required source controls; not executed", + "actions": ["mutate-prefix-case-segments-padding-whitespace-or-base64-alphabet"], + "expected": {"mutated_frames_accepted": 0, "new_claims": 0} + }, + { + "id": "frame-size-boundaries", + "scope": "required source controls; not executed", + "actions": ["exercise-outer-8192-decoded-40-and-6040-and-plaintext-6000-boundaries"], + "expected": {"over_limit_allocation_or_decryption": false, "maximum_32_byte_key_id_frame_bytes": 8092} + }, + { + "id": "aad-domain-ambiguity", + "scope": "required source controls; not executed", + "actions": ["change-prefix-schema-version-or-key-id-in-structured-aad"], + "expected": {"packets_authenticated": 0, "alternate_plaintext_parser_used": false} + }, + { + "id": "plaintext-closed-schema", + "scope": "required source controls; not executed", + "actions": ["authenticate-test-plaintext-with-duplicate-unknown-missing-or-wrong-type-field"], + "expected": {"new_claims": 0, "alternate_parser_used": false} + }, + { + "id": "keyring-closed-schema", + "scope": "required source controls; not executed", + "actions": ["supply-oversized-container-or-duplicate-unknown-entry-field"], + "expected": {"issuance_supported": false, "content_in_diagnostic": false} + }, + { + "id": "keyring-canonical-key-encoding", + "scope": "required source controls; not executed", + "actions": ["supply-alternate-padding-alphabet-or-noncanonical-key-base64"], + "expected": {"configured_keys_accepted": 0, "decoded_entropy_text_scanned": false} + }, + { + "id": "keyring-duplicate-ids-and-limit", + "scope": "required source controls; not executed", + "actions": ["supply-duplicate-id-or-fifth-entry"], + "expected": {"configuration_accepted": false, "existing_key_evicted": false} + }, + { + "id": "keyring-readonly-key-original-text", + "scope": "required source controls; not executed", + "actions": ["supply-malformed-original-text-in-nonissuing-key"], + "expected": {"configuration_accepted": false, "role_selection_before_validation": false} + }, + { + "id": "keyring-zeroization-lockgraph", + "scope": "required source controls; not executed", + "actions": ["inspect-exact-locked-graph-feature-and-key-buffer-lifecycle-controls"], + "expected": {"reduced_round_feature_enabled": false, "supported_zeroization_enabled": true, "revealing_debug_or_serialization": false} + }, + { + "id": "read-key-missing-live-startup", + "scope": "required source controls; not executed", + "actions": ["accept-and-commit", "restart-with-required-read-key-missing"], + "expected": {"issuance_supported": false, "legacy_fallback": false, "claim_silently_stranded": false} + }, + { + "id": "read-key-missing-live-reload", + "scope": "required source controls; not executed", + "actions": ["accept-and-commit", "reload-with-required-read-key-missing"], + "expected": {"issuance_supported": false, "key_automatically_evicted": false} + }, + { + "id": "rotation-renewal-replay", + "scope": "required source controls; not executed", + "actions": ["accept-with-old-key", "stop-issuing-old-key", "renew-claim", "attempt-old-read-key-removal", "replay-original-packet"], + "expected": {"required_read_key_retained": true, "same_task_owner_attempt": true} + }, + { + "id": "rotation-unaccepted-grace", + "scope": "required source controls; not executed", + "actions": ["stop-issuing-retiring-key", "attempt-removal-before-maximum-offer-lifetime-plus-admitted-skew"], + "expected": {"required_read_key_retained": true, "undocumented_clock_assumption": false} + }, + { + "id": "keyslot-exhaustion", + "scope": "required source controls; not executed", + "actions": ["retain-four-required-read-keys", "request-fifth-issuing-key"], + "expected": {"configuration_accepted": false, "required_key_evicted": false} + }, + { + "id": "no-support-before-qualification", + "scope": "required source controls; not executed", + "actions": ["check-support-before-complete-ring-durable-key-reference-migration-and-backend-qualification"], + "expected": {"supported_advertised": false} + }, + { + "id": "content-free-refusal", + "scope": "required source controls; not executed", + "actions": ["exercise-authentication-decoding-key-lookup-load-and-rng-errors"], + "expected": {"token_key_plaintext_digest_or_authority_values_in_diagnostics": false} + }, + { + "id": "keyring-presence-and-raw-size", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["omit-setting", "supply-empty-setting", "supply-invalid-utf8", "supply-valid-container-at-4096-bytes", "supply-container-at-4097-bytes"], + "expected": {"absence_supported": false, "empty_or_invalid_utf8_accepted": false, "raw_limit_checked_before_parse": true, "maximum_raw_utf8_bytes": 4096} + }, + { + "id": "keyring-container-version-and-shape", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["supply-wrong-or-missing-schema", "supply-duplicate-or-unknown-container-field", "supply-unordered-object-instead-of-keys-array", "supply-empty-array-or-wrong-entry-type"], + "expected": {"configuration_accepted": false, "alternate_or_legacy_parser_used": false} + }, + { + "id": "keyring-id-byte-boundaries", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["exercise-valid-ascii-id-at-1-and-32-bytes", "exercise-empty-33-byte-or-nonascii-id", "exercise-case-distinct-ids"], + "expected": {"valid_id_byte_range": [1, 32], "invalid_id_accepted": false, "id_case_folded": false} + }, + { + "id": "keyring-key-text-unused-bits", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["supply-44-byte-standard-base64-with-nonzero-unused-bits", "supply-url-safe-substitution-or-wrong-padding", "supply-canonical-32-byte-synthetic-key"], + "expected": {"noncanonical_key_accepted": false, "canonical_key_bytes": 32, "canonical_text_bytes": 44, "canonical_padding_bytes": 1} + }, + { + "id": "keyring-ordered-two-key-read", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["configure-two-distinct-valid-synthetic-keys", "issue-packet", "read-test-packets-under-either-retained-id"], + "expected": {"issuing_key_index": 0, "both_read_keys_usable": true, "every_original_text_validated_before_selection": true, "decoded_entropy_text_scanned": false} + } + ] +} From 5298d20bfb5e8a4fa1d8d3c277347f57fb56c15a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:50:57 +0000 Subject: [PATCH 06/12] Keep selected kind sets independent of caller references --- src/Worker/TaskDelivery.php | 73 ++++++++++++++++----------- tests/WorkerTaskDeliveryKindsTest.php | 17 +++++++ 2 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index 272adcc9..f813551f 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -88,38 +88,49 @@ public function poll(string $path, array $body, Closure $send): ?array // Hold ownership across both requests, including a suspended offer read. $this->accepting[$key] = $allowedKinds; try { - try { - if ($selected === null) { - $wirePath = substr($path, 0, -5).'/offers'; - $body['registration_token'] = $registration['token']; - $response = $send($wirePath, $body, null); - if (!array_key_exists('task_offer', $response)) { - return $response; - } - $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, $allowedKinds); - $selected = $this->pending[$key] ??= $candidate; - } - $response = $send($selected->path, $selected->requestBody(), $selected); - if ($response['task'] === null && !PollResponse::isTerminal($response)) { - throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); - } - } catch (ServerException $exception) { - $response = $this->refusedResponse($exception, $workerId); - if ($response === null) { - if ($this->isUncertain($exception, $selected)) { - throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); - } - throw $exception; - } - } - $this->resolved($key, $selected); + $body['registration_token'] = $registration['token']; - return $response; + return $this->deliver($key, substr($path, 0, -5).'/offers', $body, $allowedKinds, $selected, $send); } finally { unset($this->accepting[$key]); } } + /** + * @param array $body + * @param list $allowedKinds + * @param Closure(string, array, ?TaskDeliveryAttempt): array $send + * @return array + */ + private function deliver(string $key, string $wirePath, array $body, array $allowedKinds, ?TaskDeliveryAttempt $selected, Closure $send): array + { + try { + if ($selected === null) { + $response = $send($wirePath, $body, null); + if (!array_key_exists('task_offer', $response)) { + return $response; + } + $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, $allowedKinds); + $selected = $this->pending[$key] ??= $candidate; + } + $response = $send($selected->path, $selected->requestBody(), $selected); + if ($response['task'] === null && !PollResponse::isTerminal($response)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); + } + } catch (ServerException $exception) { + $response = $this->refusedResponse($exception, (string) $body['worker_id']); + if ($response === null) { + if ($this->isUncertain($exception, $selected)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); + } + throw $exception; + } + } + $this->resolved($key, $selected); + + return $response; + } + /** * Validate before payload hydration can initialize a claim supervisor. * @param array $response @@ -160,16 +171,18 @@ private static function requestedKinds(string $path, array $body): array if (!is_array($requested) || !array_is_list($requested) || $requested === []) { throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); } + $normalized = []; foreach ($requested as $value) { if (!is_string($value) || !in_array($value, $permitted, true)) { throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); } + // Copy scalar values rather than retaining caller-owned array references. + $normalized[] = $value; } - /** @var list $requested */ - $requested = array_values(array_unique($requested)); - sort($requested, SORT_STRING); + $normalized = array_values(array_unique($normalized)); + sort($normalized, SORT_STRING); - return $requested; + return $normalized; } public function forget(string $workerId): void diff --git a/tests/WorkerTaskDeliveryKindsTest.php b/tests/WorkerTaskDeliveryKindsTest.php index 16c0a2d6..fe073413 100644 --- a/tests/WorkerTaskDeliveryKindsTest.php +++ b/tests/WorkerTaskDeliveryKindsTest.php @@ -160,6 +160,23 @@ public function testChangedKindsCannotReplaceAnUncertainOriginalRequest(string $ /** @return array */ public static function selectedKinds(): array { return [['workflow'], ['update_validation']]; } + public function testCallerReferencesCannotMutateTheSelectedKindSetOrRetryBytes(): void + { + $kind = 'workflow'; + $requested = [&$kind, 'update_validation']; + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Timeout.', 504), self::claimed('workflow')]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, $requested)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + $originalPacket = json_encode($transport->requests[2], JSON_THROW_ON_ERROR); + $kind = 'activity'; + [$result, $error] = self::capture(static fn (): array => self::poll(clone $client, self::BOTH)); + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame(self::claimed('workflow'), $result); + self::assertSame($originalPacket, json_encode($transport->requests[3], JSON_THROW_ON_ERROR)); + } + #[DataProvider('concurrentKinds')] public function testKindSetBindingSpansOfferAndAcceptanceOwnership(string $kind, string $phase, bool $clone, bool $changed): void { From ff6b4b042636e1050b654347df4c2031c3e454fd Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:14:26 +0000 Subject: [PATCH 07/12] Require closed original-byte task acceptance refusals --- docs/portal/build/workers.md | 3 + docs/protocol.md | 20 +- src/Client.php | 3 +- src/Exception/TransportException.php | 2 + src/Transport/Psr18Transport.php | 24 +- src/Transport/TaskDeliveryRefusal.php | 45 + src/Transport/UniqueJson.php | 128 + src/Worker/TaskDelivery.php | 36 +- src/Worker/TaskDeliveryRefusal.php | 28 + tests/WorkerTaskDeliveryRefusalTest.php | 397 +++ tests/WorkerTaskDeliveryTest.php | 18 +- tests/WorkerTaskDeliveryWireTest.php | 186 ++ .../fixtures/task-delivery-contract/README.md | 23 + .../acceptance-refusals.v1.json | 2296 +++++++++++++++++ 14 files changed, 3176 insertions(+), 33 deletions(-) create mode 100644 src/Transport/TaskDeliveryRefusal.php create mode 100644 src/Transport/UniqueJson.php create mode 100644 src/Worker/TaskDeliveryRefusal.php create mode 100644 tests/WorkerTaskDeliveryRefusalTest.php create mode 100644 tests/WorkerTaskDeliveryWireTest.php create mode 100644 tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index ca88fbc5..8544c1d9 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -106,6 +106,9 @@ If that response is lost, it retries the same packet and poll identity until the Server confirms the claim, expiry, or loss of registration authority. An older node returning an error cannot switch that poll to legacy delivery. Servers without this capability continue using the existing poll protocol. +During acceptance, stop/drain hints and malformed replies retain that packet. +Only the protocol's two closed HTTP `409` refusal shapes can release it or end +the original registration; a later replacement registration is preserved. With the default cURL transport, supporting Servers send progress during idle worker polls. Progress keeps an idle connection active while the complete diff --git a/docs/protocol.md b/docs/protocol.md index 7814f0c1..7c9714cf 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -205,8 +205,24 @@ through uncertain replies. Managed workers retain this state across client clones and honor the normal retry backoff. If using the low-level `poll*TaskResponse()` methods, catch `TaskDeliveryUncertain` and retry with its `details['poll_request_id']` on the same Client. An ordinary empty response to -an acceptance does not prove that no claim committed. A confirmed offer expiry -permits a new poll; lost registration authority stops that worker. +an acceptance does not prove that no claim committed. Definitive refusal requires +HTTP `409` and exactly `task`, `poll_status`, and `reason`, with literal +`task: null`. The only accepted status/reason pairs are +`task_offer_expired` / `task_offer_expired` and +`stale_worker_registration` / `worker_registration_lost_authority`. +Expiry releases only the selected packet; authority loss ends only its original +registration incarnation. Other pending requests and replacement registrations +remain separate. The Server must establish these outcomes from receipt-first +reconciliation and authoritative claim/registration state. + +The default transport checks the complete original UTF-8 JSON response for +duplicate decoded member names before associative decoding. This also applies +inside a progress envelope. Invalid bytes, trailing data, extra fields, mixed +pairs, conflicting exception fields and successful terminal hints retain the +selected packet without running a handler. Discovery before selection keeps its +existing poll classification. A custom transport must provide original-byte +validation through `Transport\TaskDeliveryRefusal::fromJson()` and attach that +evidence to `TransportException`; a decoded-only refusal remains uncertain. Before replaying or completing a workflow task, the managed worker requires a successful lease-renewal response whose task ID, attempt, and lease owner match diff --git a/src/Client.php b/src/Client.php index 8e4771bd..e4400dfb 100644 --- a/src/Client.php +++ b/src/Client.php @@ -3017,7 +3017,8 @@ private function performRequest( throw $exception; } $details = $exception->response; - $reason = is_array($details) && isset($details['reason']) ? (string) $details['reason'] : null; + $reason = is_array($details) && isset($details['reason']) + && ($responseValidator === null || is_string($details['reason'])) ? (string) $details['reason'] : null; $status = $exception->status ?? 0; $message = $exception->getMessage(); if ($operation === 'query') { diff --git a/src/Exception/TransportException.php b/src/Exception/TransportException.php index c21f4640..ad7c1cb2 100644 --- a/src/Exception/TransportException.php +++ b/src/Exception/TransportException.php @@ -5,6 +5,7 @@ namespace DurableWorkflow\Exception; use Throwable; +use DurableWorkflow\Transport\TaskDeliveryRefusal; class TransportException extends DurableWorkflowException { @@ -17,6 +18,7 @@ public function __construct( public readonly bool $transientConnectionFailure = false, /** @var array */ public readonly array $headers = [], + public readonly ?TaskDeliveryRefusal $taskDeliveryRefusal = null, ) { parent::__construct($message, $status ?? 0, $previous); } diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index 8cc679e2..40bc0c62 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -145,7 +145,11 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); $this->responseStatus = $response->getStatusCode(); - return $this->decodeJsonResponse($response, $poll); + $acceptance = PollTransport::taskKind($method, $uri) !== null + && str_ends_with((string) parse_url($uri, PHP_URL_PATH), '/offers') + && is_string($body['task_offer_token'] ?? null) && $body['task_offer_token'] !== ''; + + return $this->decodeJsonResponse($response, $poll, $acceptance); } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -169,15 +173,16 @@ private function pollTimeoutSeconds(?array $body): int|float } /** @return array|list|null */ - private function decodeJsonResponse(ResponseInterface $response, bool $poll = false): ?array + private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, bool $acceptance = false): ?array { $rawBody = (string) $response->getBody(); $status = $response->getStatusCode(); $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); + $proof = $acceptance ? TaskDeliveryRefusal::fromJson($status, $rawBody, $poll && $mediaType === PollTransport::MEDIA_TYPE) : null; if ($poll && $mediaType === PollTransport::MEDIA_TYPE) { $frame = PollTransport::decode($rawBody, $status); if ($frame['status'] >= 300) { - throw TransportException::fromResponse($frame['status'], $frame['body'], $rawBody, $frame['headers']); + throw $this->responseFailure($frame['status'], $frame['body'], $rawBody, $frame['headers'], $acceptance, $proof); } return $frame['body']; @@ -186,7 +191,7 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // An upstream proxy can return HTML or an empty body. Preserve // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); - throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); + throw $this->responseFailure($status, is_array($decoded) ? $decoded : null, $rawBody, [], $acceptance, $proof); } $decoded = $rawBody === '' ? null : json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR); @@ -197,6 +202,17 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa return $decoded; } + /** + * @param array|list|null $body + * @param array $headers + */ + private function responseFailure(int $status, ?array $body, string $raw, array $headers, bool $acceptance, ?TaskDeliveryRefusal $proof): TransportException + { + return $acceptance + ? new TransportException('Task acceptance is not confirmed.', $status, $body, headers: $headers, taskDeliveryRefusal: $proof) + : TransportException::fromResponse($status, $body, $raw, $headers); + } + /** * @param array $headers * @param array|null $body diff --git a/src/Transport/TaskDeliveryRefusal.php b/src/Transport/TaskDeliveryRefusal.php new file mode 100644 index 00000000..574a8d9f --- /dev/null +++ b/src/Transport/TaskDeliveryRefusal.php @@ -0,0 +1,45 @@ + 'task_offer_expired', + 'stale_worker_registration' => 'worker_registration_lost_authority', + ]; + + private function __construct( + public readonly int $status, + public readonly string $pollStatus, + public readonly string $reason, + ) { + } + + public static function fromJson(int $status, string $raw, bool $progress = false): ?self + { + $body = UniqueJson::decode($raw); + if ($progress) { + $frame = PollTransport::decode($raw, $status); + $status = $frame['status']; + $body = $frame['body']; + } + if ($status !== 409 || !is_array($body) || count($body) !== 3 + || !array_key_exists('task', $body) || $body['task'] !== null + || !is_string($body['poll_status'] ?? null) || !is_string($body['reason'] ?? null) + || (self::PAIRS[$body['poll_status']] ?? null) !== $body['reason']) { + return null; + } + + return new self($status, $body['poll_status'], $body['reason']); + } + + /** @return array{task: null, poll_status: string, reason: string} */ + public function body(): array + { + return ['task' => null, 'poll_status' => $this->pollStatus, 'reason' => $this->reason]; + } +} diff --git a/src/Transport/UniqueJson.php b/src/Transport/UniqueJson.php new file mode 100644 index 00000000..a8c7611f --- /dev/null +++ b/src/Transport/UniqueJson.php @@ -0,0 +1,128 @@ += 512) { + throw new JsonException('Maximum JSON depth exceeded.'); + } + self::whitespace($json, $offset); + $token = $json[$offset] ?? ''; + if ($token === '{') { + self::object($json, $offset, $depth + 1); + } elseif ($token === '[') { + self::items($json, $offset, $depth + 1); + } elseif ($token === '"') { + self::stringToken($json, $offset); + } elseif (preg_match('/\G(?:true|false|null|-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)/A', $json, $match, 0, $offset) === 1) { + $offset += strlen($match[0]); + } else { + throw new JsonException('Invalid JSON value.'); + } + } + + private static function object(string $json, int &$offset, int $depth): void + { + ++$offset; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') === '}') { + ++$offset; + return; + } + $seen = []; + while (true) { + if (($json[$offset] ?? '') !== '"') { + throw new JsonException('Invalid JSON object member.'); + } + $key = json_decode(self::stringToken($json, $offset), true, 512, JSON_THROW_ON_ERROR); + $identity = "\0".$key; + if (isset($seen[$identity])) { + throw new JsonException('Duplicate JSON object member.'); + } + $seen[$identity] = true; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') !== ':') { + throw new JsonException('Invalid JSON object separator.'); + } + ++$offset; + self::value($json, $offset, $depth); + self::whitespace($json, $offset); + $separator = $json[$offset] ?? ''; + ++$offset; + if ($separator === '}') { + return; + } + if ($separator !== ',') { + throw new JsonException('Invalid JSON object separator.'); + } + self::whitespace($json, $offset); + } + } + + private static function items(string $json, int &$offset, int $depth): void + { + ++$offset; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') === ']') { + ++$offset; + return; + } + while (true) { + self::value($json, $offset, $depth); + self::whitespace($json, $offset); + $separator = $json[$offset] ?? ''; + ++$offset; + if ($separator === ']') { + return; + } + if ($separator !== ',') { + throw new JsonException('Invalid JSON array separator.'); + } + } + } + + private static function stringToken(string $json, int &$offset): string + { + $start = $offset++; + while (($end = strpos($json, '"', $offset)) !== false) { + $slashes = 0; + for ($index = $end - 1; $index > $start && $json[$index] === '\\'; --$index) { + ++$slashes; + } + $offset = $end + 1; + if ($slashes % 2 === 0) { + return substr($json, $start, $offset - $start); + } + } + + throw new JsonException('Unterminated JSON string.'); + } + + private static function whitespace(string $json, int &$offset): void + { + $offset += strspn($json, " \t\r\n", $offset); + } +} diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index f813551f..a845e790 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -114,11 +114,11 @@ private function deliver(string $key, string $wirePath, array $body, array $allo $selected = $this->pending[$key] ??= $candidate; } $response = $send($selected->path, $selected->requestBody(), $selected); - if ($response['task'] === null && !PollResponse::isTerminal($response)) { + if ($response['task'] === null) { throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); } } catch (ServerException $exception) { - $response = $this->refusedResponse($exception, (string) $body['worker_id']); + $response = $selected === null ? null : $this->refusedResponse($exception, $selected->requestBody()); if ($response === null) { if ($this->isUncertain($exception, $selected)) { throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); @@ -201,28 +201,25 @@ public function __debugInfo(): array return ['registered_workers' => count($this->registrations), 'unresolved_deliveries' => count($this->pending)]; } - private function definitiveRefusal(ServerException $exception): bool - { - $response = $exception->details; - - return $exception->status === 409 && $response !== null && !array_is_list($response) - && array_key_exists('task', $response) && $response['task'] === null - && ($response['reason'] ?? null) === $exception->reason - && (in_array($exception->reason, ['task_offer_expired', 'worker_registration_lost_authority'], true) - || PollResponse::isTerminal($response)); - } - - /** @return array|null */ - private function refusedResponse(ServerException $exception, string $workerId): ?array + /** + * @param array $request + * @return array|null + */ + private function refusedResponse(ServerException $exception, array $request): ?array { - if (!$this->definitiveRefusal($exception)) { + if (!TaskDeliveryRefusal::matches($exception)) { return null; } $response = $exception->details ?? []; if ($exception->reason === 'task_offer_expired') { $response['poll_status'] = 'empty'; } else { - $this->forget($workerId); + $workerId = (string) $request['worker_id']; + $current = $this->registrations[$workerId] ?? null; + if ($current !== null && $current['token'] === ($request['registration_token'] ?? null) + && $current['queue'] === ($request['task_queue'] ?? null)) { + $this->forget($workerId); + } } return $response; @@ -231,10 +228,7 @@ private function refusedResponse(ServerException $exception, string $workerId): private function isUncertain(ServerException $exception, ?TaskDeliveryAttempt $selected): bool { return !$exception instanceof TaskDeliveryUncertain && !PollResponse::isTransientPayloadFailure($exception) - && (($selected !== null && ($exception->status >= 500 || $exception->status === 409 || $exception->status === 429 - || $exception->reason === 'invalid_task_delivery_response' - || in_array($exception->status, [0, 404, 405], true))) - || ($selected === null && in_array($exception->status, [404, 405], true))); + && ($selected !== null || in_array($exception->status, [404, 405], true)); } private function resolved(string $key, ?TaskDeliveryAttempt $selected): void diff --git a/src/Worker/TaskDeliveryRefusal.php b/src/Worker/TaskDeliveryRefusal.php new file mode 100644 index 00000000..2cb5b295 --- /dev/null +++ b/src/Worker/TaskDeliveryRefusal.php @@ -0,0 +1,28 @@ +getPrevious(); + if (!$transport instanceof TransportException || $transport->taskDeliveryRefusal === null) { + return false; + } + $proof = $transport->taskDeliveryRefusal; + $response = $exception->details; + + return $transport->status === $proof->status && $exception->status === $proof->status + && $exception->reason === $proof->reason && $response !== null && count($response) === 3 + && array_key_exists('task', $response) && $response['task'] === null + && ($response['poll_status'] ?? null) === $proof->pollStatus + && ($response['reason'] ?? null) === $proof->reason; + } +} diff --git a/tests/WorkerTaskDeliveryRefusalTest.php b/tests/WorkerTaskDeliveryRefusalTest.php new file mode 100644 index 00000000..2ae3fb0e --- /dev/null +++ b/tests/WorkerTaskDeliveryRefusalTest.php @@ -0,0 +1,397 @@ + $response */ + #[DataProvider('malformedRefusals')] + public function testMalformed409RetainsThePacketAndNegotiatedRegistration(array $response): void + { + self::assertUncertainReply(self::wireFailure($response)); + } + + /** @return iterable}> */ + public static function malformedRefusals(): iterable + { + $pairs = self::existingPairs(); + $statuses = array_values(array_unique([...array_column($pairs, 0), 'draining', 'stopped', + 'worker_heartbeat_stale', 'worker_registration_lost_authority', 'empty', 'offered', 'foreign_status'])); + $reasons = array_values(array_unique([...array_column($pairs, 1), 'worker_draining', 'worker_stopped', + 'stale_worker_registration', 'worker_heartbeat_stale', 'backend_unavailable', 'unrelated_reason'])); + foreach ($statuses as $status) { + foreach ($reasons as $reason) { + if (!in_array([$status, $reason], $pairs, true)) { + yield $status.' / '.$reason => [self::body($status, $reason)]; + } + } + } + foreach ($pairs as [$status, $reason]) { + foreach (self::contradictoryFields() as $name => [$key, $value]) { + yield $status.' / '.$name => [[...self::body($status, $reason), $key => $value]]; + } + foreach (['task', 'poll_status', 'reason'] as $key) { + $body = self::body($status, $reason); + unset($body[$key]); + yield $status.' / missing '.$key => [$body]; + } + } + } + + /** @return array */ + private static function contradictoryFields(): array + { + return [ + 'non-null task' => ['task', ['task_id' => 'unexpected']], + 'false task' => ['task', false], + 'offer null is still present' => ['task_offer', null], + 'offer object' => ['task_offer', ['task_id' => 'other-task']], + 'acceptance token' => ['task_offer_token', 'contradictory-packet'], + 'accepted marker' => ['accepted', true], + 'lease owner' => ['lease_owner', 'worker'], + 'attempt marker' => ['attempt', 1], + 'task ID outside task' => ['task_id', 'selected-task'], + 'kind outside task' => ['task_kind', 'workflow'], + 'claim retry marker' => ['retry_same_poll_request_id', true], + 'retryable wrong type' => ['retryable', 'true'], + 'retryable true' => ['retryable', true], + 'retryable false' => ['retryable', false], + 'retry delay' => ['retry_after_seconds', 1], + 'matching worker' => ['worker_id', 'worker'], + 'matching queue' => ['task_queue', 'orders'], + 'plain error text' => ['error', 'Synthetic refusal.'], + 'plain message text' => ['message', 'Synthetic refusal.'], + 'null build' => ['registered_build_id', null], + 'unknown extension' => ['unknown_terminal_extension', true], + 'foreign worker' => ['worker_id', 'other-worker'], + 'foreign queue' => ['task_queue', 'other-queue'], + 'worker wrong type' => ['worker_id', 1], + 'queue wrong type' => ['task_queue', null], + 'error wrong type' => ['error', []], + 'message wrong type' => ['message', []], + 'build wrong type' => ['registered_build_id', []], + 'status wrong type' => ['worker_status', []], + 'drain intent wrong type' => ['drain_intent', []], + 'null poll status' => ['poll_status', null], + 'null reason' => ['reason', null], + ]; + } + + /** @param array $response */ + #[DataProvider('unqualifiedSuccessReplies')] + public function testAnUnqualified200CannotResolveASelectedAcceptance(array $response): void + { + self::assertUncertainReply($response); + } + + /** @return iterable}> */ + public static function unqualifiedSuccessReplies(): iterable + { + yield 'mixed successful terminal' => [self::body('draining', 'unrelated_reason')]; + yield 'successful expiry is not the required 409' => [self::body('task_offer_expired', 'task_offer_expired')]; + yield 'successful authority loss is not the required 409' => [self::body('stale_worker_registration', 'worker_registration_lost_authority')]; + yield 'successful stopped hint' => [self::body('stopped', 'worker_stopped')]; + yield 'successful draining hint' => [self::body('draining', 'worker_draining')]; + yield 'bare legacy stale is not definitive acceptance' => [['task' => null, 'poll_status' => 'stale_worker_registration']]; + } + + #[DataProvider('wrongHttpStatuses')] + public function testAnotherHttpStatusCannotResolveTheSelectedPacket(int $status, string $pollStatus, string $reason): void + { + $body = self::body($pollStatus, $reason); + self::assertUncertainReply($status === 200 ? $body : new TransportException('Synthetic refusal.', $status, $body)); + } + + /** @return iterable */ + public static function wrongHttpStatuses(): iterable + { + foreach (self::existingPairs() as [$status, $reason]) { + foreach ([0, 200, 202, 400, 401, 403, 404, 405, 410, 429, 500, 503] as $http) { + yield $status.' / '.$http => [$http, $status, $reason]; + } + } + } + + public function testExceptionAndBodyReasonsMustAgree(): void + { + self::assertUncertainReply(new ServerException('Synthetic refusal.', 409, 'unrelated_reason', + self::body('task_offer_expired', 'task_offer_expired'), self::wireFailure(self::body('task_offer_expired', 'task_offer_expired')))); + } + + #[DataProvider('positiveRefusals')] + public function testTheExactDefinitiveRefusalPermitsOnlyItsDefinedTransition(string $status, string $reason): void + { + $body = self::body($status, $reason); + $transport = new FakeTransport([self::registration(), self::offer(), self::wireFailure($body), + self::registration(), self::offer('new-packet'), self::claimed()]); + $client = self::client($transport); + $expected = $body; + if ($reason === 'task_offer_expired') { $expected['poll_status'] = 'empty'; } + self::assertSame($expected, self::poll($client)); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed(), self::poll($client, 'new-poll')); + self::assertCount(6, $transport->requests); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[4]['body']); + self::assertSame('new-poll', $transport->requests[4]['body']['poll_request_id']); + self::assertSame('new-packet', $transport->requests[5]['body']['task_offer_token']); + } + + /** @return list */ + public static function positiveRefusals(): array { return self::existingPairs(); } + + public function testExpiryPreservesRegistrationAndAnotherUnresolvedPacket(): void + { + [$client, $transport] = self::pendingClient('poll-a', self::wireFailure(self::body('task_offer_expired', 'task_offer_expired'))); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b'); + self::assertSame('empty', self::poll($client, 'poll-a')['poll_status']); + self::assertSame(self::claimed('poll-b'), self::poll(clone $client, 'poll-b')); + self::assertSame(self::claimed('poll-c'), self::poll($client, 'poll-c')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-c', false)); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testAnUncertainLifecycleReplyCannotEraseAnotherPendingPacket(): void + { + [$client, $transport] = self::pendingClient('poll-a', new TransportException('Draining.', 409, + self::body('draining', 'worker_draining'))); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b'); + self::expectPending($client, 'poll-a'); + self::assertSame(self::claimed('poll-b'), self::poll(clone $client, 'poll-b')); + self::assertSame(self::claimed('poll-a'), self::poll($client, 'poll-a')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertIdenticalRetries($transport, 'poll-a', 3); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testDiscoveryDrainingDoesNotEraseAnAlreadySelectedPacket(): void + { + $stop = self::body('draining', 'worker_draining'); + [$client, $transport] = self::pendingClient('discovery', new TransportException('Draining.', 409, $stop), true); + self::expectPending($client, 'poll-a'); + self::assertSame($stop, self::poll($client, 'discovery')); + self::assertSame(self::claimed('poll-a'), self::poll(clone $client, 'poll-a')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertIdenticalRetries($transport, 'poll-a', 2); + } + + public function testProvedAuthorityLossDoesNotForgetAnotherWorker(): void + { + $lost = self::body('stale_worker_registration', 'worker_registration_lost_authority'); + [$client, $transport] = self::pendingClient('poll-a', self::wireFailure($lost)); + $client->registerWorker('worker-b', 'orders', [], []); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b', 'worker-b'); + self::assertSame($lost, self::poll($client, 'poll-a')); + self::assertSame(self::claimed('poll-b', 'worker-b'), self::poll(clone $client, 'poll-b', 'worker-b')); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testTheLegacyBareStalePollContractIsPreserved(): void + { + $response = ['task' => null, 'poll_status' => 'stale_worker_registration']; + $transport = new FakeTransport([['registered' => true], $response]); + $client = self::client($transport); + self::assertSame($response, self::poll($client)); + self::assertTrue(PollResponse::isTerminal($response)); + self::assertStringEndsWith('/workflow-tasks/poll', $transport->requests[1]['uri']); + } + + /** @param TransportException|ServerException|array $reply */ + private static function assertUncertainReply(TransportException|ServerException|array $reply): void + { + $steps = [self::registration(), self::offer(), $reply, self::claimed()]; + $transport = new FakeTransport(handler: static function () use (&$steps): ?array { + $next = array_shift($steps); + if ($next instanceof TransportException || $next instanceof ServerException) { throw $next; } + return $next; + }); + $client = self::client($transport); + $caught = null; + try { self::poll($client); } + catch (ServerException $error) { $caught = $error; } + self::assertInstanceOf(TaskDeliveryUncertain::class, $caught); + self::assertTrue(PollResponse::isTransientFailure($caught), 'An uncertain reply cannot terminate the managed worker.'); + self::assertSame(self::claimed(), self::poll(clone $client)); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertStringEndsWith('/workflow-tasks/offers', $transport->requests[3]['uri']); + } + + /** @param array $body */ + private static function wireFailure(array $body): TransportException + { + return new TransportException('Synthetic refusal.', 409, $body, + taskDeliveryRefusal: \DurableWorkflow\Transport\TaskDeliveryRefusal::fromJson(409, json_encode($body, JSON_THROW_ON_ERROR))); + } + + public function testADecodedOnlyRefusalCannotResolveThePacket(): void + { + self::assertUncertainReply(new TransportException('No original-byte evidence.', 409, + self::body('task_offer_expired', 'task_offer_expired'))); + } + + public function testAnOldAuthorityRefusalCannotEraseAReplacementIncarnation(): void + { + $registrations = 0; + $attempts = []; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use (&$registrations, &$attempts): array { + if (str_ends_with($uri, '/worker/register')) { + $response = self::registration(); + $response['registration_token'] = str_repeat(++$registrations === 1 ? 'a' : 'b', 32); + return $response; + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { return self::offer('packet-'.$poll, $poll); } + if ($poll === 'old') { + \Fiber::suspend(); + throw self::wireFailure(self::body('stale_worker_registration', 'worker_registration_lost_authority')); + } + if (!isset($attempts[$poll])) { + $attempts[$poll] = true; + throw new TransportException('Lost response.', 504); + } + return self::claimed($poll); + }); + $client = self::client($transport); + $old = new \Fiber(static fn (): array => self::poll($client, 'old')); + $old->start(); + self::assertTrue($old->isSuspended()); + $replacement = clone $client; + $replacement->registerWorker('worker', 'orders', [], []); + self::expectPending($replacement, 'new'); + $old->resume(); + self::assertSame('worker_registration_lost_authority', $old->getReturn()['reason']); + self::assertSame(self::claimed('new'), self::poll($replacement, 'new')); + self::assertCount(1, self::packetsFor($transport, 'new', false)); + self::assertIdenticalRetries($transport, 'new', 2); + self::assertSame(str_repeat('b', 32), self::packetsFor($transport, 'new', true)[1]['body']['registration_token']); + } + + /** @return list */ + private static function existingPairs(): array + { + // Exact two rows in the reviewed Server 31b281ac companion. + return [ + ['task_offer_expired', 'task_offer_expired'], + ['stale_worker_registration', 'worker_registration_lost_authority'], + ]; + } + + /** @return array */ + private static function body(string $status, string $reason): array + { + return ['task' => null, 'poll_status' => $status, 'reason' => $reason]; + } + + private static function client(FakeTransport $transport): Client + { + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + return $client; + } + + /** @return array */ + private static function poll(Client $client, string $pollId = 'refusal-poll', string $workerId = 'worker'): array + { + return $client->pollWorkflowTaskResponse($workerId, 'orders', 4, $pollId); + } + + private static function expectPending(Client $client, string $pollId, string $workerId = 'worker'): void + { + $caught = null; + try { self::poll($client, $pollId, $workerId); } + catch (ServerException $error) { $caught = $error; } + self::assertInstanceOf(TaskDeliveryUncertain::class, $caught); + } + + /** @return array{Client, FakeTransport} */ + private static function pendingClient(string $refusalPoll, TransportException $refusal, bool $discovery = false): array + { + $attempts = []; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ( + $refusalPoll, $refusal, $discovery, &$attempts, + ): array { + if (str_ends_with($uri, '/worker/register')) { + $response = self::registration(); + $response['registration_token'] = str_repeat($body['worker_id'] === 'worker' ? 'a' : 'b', 32); + return $response; + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { + if ($discovery && $poll === $refusalPoll) { throw $refusal; } + return self::offer('packet-'.$poll, $poll); + } + $attempts[$poll] = ($attempts[$poll] ?? 0) + 1; + if (in_array($poll, ['poll-a', 'poll-b'], true) && $attempts[$poll] === 1) { + throw new TransportException('Lost response.', 504); + } + if (!$discovery && $poll === $refusalPoll && $attempts[$poll] === 2) { throw $refusal; } + return self::claimed($poll, $body['worker_id']); + }); + + return [self::client($transport), $transport]; + } + + /** @return list> */ + private static function packetsFor(FakeTransport $transport, string $pollId, bool $accept): array + { + return array_values(array_filter($transport->requests, static fn (array $request): bool => + ($request['body']['poll_request_id'] ?? null) === $pollId + && isset($request['body']['task_offer_token']) === $accept)); + } + + private static function assertIdenticalRetries(FakeTransport $transport, string $pollId, int $count): void + { + $requests = self::packetsFor($transport, $pollId, true); + self::assertCount($count, $requests); + $first = json_encode($requests[0], JSON_THROW_ON_ERROR); + foreach ($requests as $request) { + self::assertSame($first, json_encode($request, JSON_THROW_ON_ERROR)); + } + } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, + 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $token = 'selected-packet', string $taskId = 'selected-task'): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => 'workflow', 'task_id' => $taskId, + 'token' => $token, 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $taskId = 'selected-task', string $workerId = 'worker'): array + { + return ['task' => ['task_id' => $taskId, 'lease_owner' => $workerId, 'task_kind' => 'workflow']]; + } +} diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php index 48075982..a7dd04c6 100644 --- a/tests/WorkerTaskDeliveryTest.php +++ b/tests/WorkerTaskDeliveryTest.php @@ -244,10 +244,17 @@ public function testAnOlderNodeCannotDowngradeTheNegotiatedEndpoint(): void self::assertStringEndsWith('/activity-tasks/offers', $transport->requests[2]['uri']); } + private static function refusal(string $status, string $reason): TransportException + { + $body = ['task' => null, 'poll_status' => $status, 'reason' => $reason]; + return new TransportException('Synthetic refusal.', 409, $body, + taskDeliveryRefusal: \DurableWorkflow\Transport\TaskDeliveryRefusal::fromJson(409, json_encode($body, JSON_THROW_ON_ERROR))); + } + public function testAConfirmedExpiryReleasesTheSelectedPacket(): void { $transport = new FakeTransport([self::registration(), self::offer('workflow'), - new TransportException('Expired.', 409, ['task' => null, 'poll_status' => 'task_offer_expired', 'reason' => 'task_offer_expired']), + self::refusal('task_offer_expired', 'task_offer_expired'), self::offer('workflow', 'second-packet'), self::claimed('workflow')]); $client = new Client('https://server.example', transport: $transport); $client->registerWorker('worker', 'orders', [], []); @@ -271,16 +278,17 @@ public function testRegistrationCannotDowngradeAnUnresolvedAcceptance(): void self::assertSame($transport->requests[2], $transport->requests[4]); } - public function testAnAuthoritativeWorkerStopIsNotRetriedAsAnUncertainAcceptance(): void + public function testAnAuthoritativeDiscoveryWorkerStopNeedsNoAcceptance(): void { $terminal = ['task' => null, 'poll_status' => 'draining', 'reason' => 'worker_draining']; - $transport = new FakeTransport([self::registration(), self::offer('workflow'), + $transport = new FakeTransport([self::registration(), new TransportException('Worker is draining.', 409, $terminal)]); $client = new Client('https://server.example', transport: $transport); $client->registerWorker('worker', 'orders', [], []); self::assertSame($terminal, self::poll($client, 'workflow')); self::assertTrue(PollResponse::isTerminal($terminal)); - self::assertCount(3, $transport->requests); + self::assertCount(2, $transport->requests); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[1]['body']); } public function testReplacementRegistrationDiscardsOnlyTheOldIncarnationsPacket(): void @@ -352,7 +360,7 @@ public function testTheManagedWorkerRetainsItsLogicalPollAfterAnUncertainAccepta $retries = []; $transport = new FakeTransport([self::registration(), self::offer('workflow'), new TransportException('Not found.', 404), - ['task' => null, 'poll_status' => 'stopped', 'reason' => 'worker_stopped']]); + self::refusal('stale_worker_registration', 'worker_registration_lost_authority')]); $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders', workerId: 'worker', clock: static function () use (&$now): float { return $now; }, sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, diff --git a/tests/WorkerTaskDeliveryWireTest.php b/tests/WorkerTaskDeliveryWireTest.php new file mode 100644 index 00000000..a465ff96 --- /dev/null +++ b/tests/WorkerTaskDeliveryWireTest.php @@ -0,0 +1,186 @@ + $input */ + #[DataProvider('wireCases')] + public function testTheOriginalWireReplyControlsOnlyItsSelectedAcceptance(array $input, string $action, string $endpoint, string $kind, bool $raw, bool $progress = false): void + { + $requests = []; + $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use (&$requests, $input, $raw, $kind, $progress) { + $body = json_decode((string) $request->getBody(), true, flags: JSON_THROW_ON_ERROR); + $requests[] = ['uri' => (string) $request->getUri(), 'headers' => $request->getHeaders(), 'body' => (string) $request->getBody()]; + $index = count($requests); + if ($index === 1) { return Create::promiseFor(self::json(self::registration())); } + if ($index === 2 || ($index === 4 && !isset($body['task_offer_token']) && str_ends_with((string) $request->getUri(), '/offers'))) { + return Create::promiseFor(self::json(self::offer($kind))); + } + if ($index === 3) { return Create::promiseFor(self::reply($input, $raw, $progress)); } + $reply = str_ends_with((string) $request->getUri(), '/poll') ? ['task' => null] : self::claimed($kind); + return Create::promiseFor(self::json($reply)); + }]); + $transport = new Psr18Transport(client: $http); + $client = new Client('https://server.example', transport: self::withExceptionContext($transport, $input)); + $observed = 0; + $client = $client->withWorkerPayloadObserver(static function (string $path, ?array $body, array $response) use (&$observed): ?Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + $result = null; + $error = null; + try { $result = self::poll($client, $endpoint, 'original'); } + catch (ServerException $caught) { $error = $caught; } + self::assertSame(0, $observed, 'No refusal or uncertain outcome may start task execution.'); + if ($action === 'retain_selected_packet') { + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(self::claimed($kind), self::poll(clone $client, $endpoint, 'original')); + self::assertSame(1, $observed); + self::assertCount(4, $requests); + self::assertSame($requests[2], $requests[3], 'Retry must preserve the selected packet, kind set and request.'); + return; + } + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame($action === 'resolve_selected_packet' ? 'empty' : 'stale_worker_registration', $result['poll_status']); + self::poll($client, $endpoint, 'next'); + self::assertStringEndsWith('/'.$endpoint.'-tasks/'.($action === 'resolve_selected_packet' ? 'offers' : 'poll'), $requests[3]['uri']); + self::assertArrayNotHasKey('task_offer_token', json_decode($requests[3]['body'], true, flags: JSON_THROW_ON_ERROR)); + self::assertCount($action === 'resolve_selected_packet' ? 5 : 4, $requests); + } + + /** @return iterable, string, string, string, bool, bool}> */ + public static function wireCases(): iterable + { + foreach (['wire_cases' => false, 'raw_wire_cases' => true] as $group => $raw) { + foreach (self::fixture()[$group] as $case) { + foreach ([['workflow', 'workflow'], ['workflow', 'update_validation'], ['activity', 'activity'], ['query', 'query'], ['update-validation', 'update_validation']] as [$endpoint, $kind]) { + foreach ($raw ? [false, true] : [false] as $progress) { + yield $case['id'].' / '.$endpoint.' / '.$kind.($progress ? ' / progress' : '') => [$case['input'], $case['expected_action'], $endpoint, $kind, $raw, $progress]; + } + } + } + } + } + + /** @param array $input */ + private static function reply(array $input, bool $raw, bool $progress): Response + { + if (isset($input['transport_failure'])) { throw new TransportException('Synthetic body read failure.'); } + if ($raw) { + $bytes = isset($input['raw_body_hex']) ? hex2bin($input['raw_body_hex']) : $input['raw_body_utf8']; + if ($progress) { + return new Response(200, ['Content-Type' => PollTransport::MEDIA_TYPE], + '{"schema":"durable-workflow.poll-transport.v1","status":'.$input['http_status'].',"headers":{},"body":'.$bytes.'}'); + } + return new Response($input['http_status'], ['Content-Type' => 'application/json'], $bytes); + } + // The real progress decoder also exercises wrong-type logical statuses + // that PSR-7's integer status constructor cannot represent directly. + return new Response(200, ['Content-Type' => PollTransport::MEDIA_TYPE], json_encode([ + 'schema' => 'durable-workflow.poll-transport.v1', 'status' => $input['http_status'], + 'headers' => (object) [], 'body' => $input['decoded_body'], + ], JSON_THROW_ON_ERROR)); + } + + /** @param array $input */ + private static function withExceptionContext(Psr18Transport $transport, array $input): Transport + { + return new class($transport, $input) implements Transport { + /** @param array $input */ + public function __construct(private Psr18Transport $transport, private array $input) {} + + public function send(string $method, string $uri, array $headers, ?array $body = null): ?array + { + try { return $this->transport->send($method, $uri, $headers, $body); } + catch (TransportException $error) { + $input = $this->input; + if (!array_key_exists('exception_http_status', $input) && !array_key_exists('exception_reason', $input)) { throw $error; } + $status = array_key_exists('exception_http_status', $input) ? $input['exception_http_status'] : $error->status; + $reason = array_key_exists('exception_reason', $input) ? $input['exception_reason'] : ($error->response['reason'] ?? null); + try { $context = new ServerException('Synthetic typed exception context.', $status, $reason, $error->response, $error); } + // Invalid types cannot become typed SDK exception fields. + // Treat that construction failure as unreadable, without coercion. + catch (TypeError $typeError) { throw new TransportException('Unreadable exception context.', previous: $typeError); } + throw $context; + } + } + }; + } + + /** @return array */ + private static function fixture(): array { return json_decode((string) file_get_contents(self::FIXTURE), true, flags: JSON_THROW_ON_ERROR); } + + /** @return array */ + private static function poll(Client $client, string $endpoint, string $poll): array + { + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => 'orders', 'timeout_seconds' => 4, 'poll_request_id' => $poll, + 'task_kinds' => $endpoint === 'workflow' ? ['workflow', 'update_validation'] : [str_replace('-', '_', $endpoint)], + ]); + } + + /** @param array $body */ + private static function json(array $body): Response { return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, 'version' => TaskDelivery::VERSION, + 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'selected-task', 'token' => 'synthetic-packet', 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => 'selected-task', 'task_kind' => $kind, 'lease_owner' => 'worker']]; + } +} diff --git a/tests/fixtures/task-delivery-contract/README.md b/tests/fixtures/task-delivery-contract/README.md index c261c0e8..3ac933c2 100644 --- a/tests/fixtures/task-delivery-contract/README.md +++ b/tests/fixtures/task-delivery-contract/README.md @@ -17,3 +17,26 @@ selection, clones and uncertainty. They do not execute the other native claim/receipt, seal/key, database, proxy, restart or capacity scenarios. Reading or enumerating those rows supplies no execution result. Tokens are synthetic opaque test text, not sealed native packets or real credentials. + +`acceptance-refusals.v1.json` is byte-identical to Server commit +`31b281ac3891143eae9c6c5ace9c65b63e404e17`, tree +`d0d1cf302f2076c3307d8bc71c8196d6b6d3c062`, path +`tests/Fixtures/contracts/task-delivery-offers/acceptance-refusals.v1.json`: + +- SHA-256: `64a936382d4ce829d0e1de50e75247223711bf4011c830b257143b0a56c237ae` +- Size: 74,169 bytes; 175 unique IDs: 117 decoded wire, 34 raw wire and + 24 authority scenarios. +- Exact companion source review: + https://github.com/durable-workflow/server/issues/931#issuecomment-6659734780 + +`WorkerTaskDeliveryWireTest` sends the 117 decoded wire cases through the actual +PHP progress-envelope transport and the 34 original-byte cases through both +ordinary and progress-envelope responses. Each runs for workflow, mixed-kind +workflow selecting update validation, activity, query and dedicated update +validation. Exception-context cases preserve each present field; fields that +cannot be represented by PHP's strict exception constructor fail as unreadable, +without coercion. Invalid UTF-8 hex is decoded byte-for-byte before transport. +Supplementary client-state controls cover clones, pending-request isolation and +replacement incarnations. These are in-memory PHP source tests. The 24 Server +authority/receipt/race scenarios are inventoried only; neither those requirements +nor native transactions, runtime behavior or installed packages are qualified. diff --git a/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json b/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json new file mode 100644 index 00000000..e466f916 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json @@ -0,0 +1,2296 @@ +{ + "schema": "durable-workflow.task-delivery-acceptance-refusals.v1", + "status": "source contract data; no classifier, executor, capability or qualification result", + "source_issue": "https://github.com/durable-workflow/server/issues/931", + "decisions": { + "general": "https://github.com/durable-workflow/server/issues/931#issuecomment-6659034006", + "security_design": "https://github.com/durable-workflow/server/issues/931#issuecomment-6659043856" + }, + "parent_fixture": { + "commit": "955da4e870ef76e7df1bcd24beaae037df63a8f4", + "path": "tests/Fixtures/contracts/task-delivery-offers/offer-accept.v1.json", + "sha256": "cac2bc6852a09db9526c623c3683772ebbba7d3f1d0b401fb92ae2026cd32b11", + "bytes": 18142, + "case_count": 49, + "unchanged_required": true + }, + "contract": { + "applies_to": "a selected acceptance packet; ordinary discovery before selection retains its separate poll contract", + "http_status": 409, + "body_type": "JSON object", + "exact_body_keys": [ + "task", + "poll_status", + "reason" + ], + "task": null, + "poll_status_type": "string", + "reason_type": "string", + "allowed_responses": [ + { + "id": "expiry", + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "action": "resolve_selected_packet", + "requires": "authoritative serialized no-acceptance proof; committed live receipt wins over deadline" + }, + { + "id": "authority_loss", + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "action": "end_original_registration", + "requires": "authoritative proof original incarnation can no longer execute; mode/freshness label alone is insufficient" + } + ], + "available_exception_fields_must_match_response": true, + "raw_response_before_ordinary_decode": { + "strict_utf8_json_required": true, + "exactly_one_complete_json_value_required": true, + "only_json_whitespace_may_follow_value": true, + "partial_prefix_parsing_or_trailing_payload_allowed": false, + "duplicate_decoded_member_names_forbidden": true, + "duplicate_identical_values_also_forbidden": true, + "last_member_wins_or_first_member_wins_not_sufficient": true, + "unavailable_raw_uniqueness_proof": "retain_selected_packet", + "unique_escaped_member_names": "ordinary JSON decoding applies; compare decoded distinct names and exact scalar values" + }, + "normalization_or_aliases_allowed": false, + "other_response_action": "retain_selected_packet", + "uncertainty_invariants": { + "original_packet_endpoint_registration_poll_and_kind_set_preserved": true, + "unrelated_pending_packets_preserved": true, + "payload_observation_hydration_execution_authorized": false, + "new_logical_poll_authorized": false, + "legacy_claim_authorized": false, + "end_original_registration_authorized": false + }, + "per_response_scope": "expiry resolves only selected packet; proved authority loss ends only the original registration, never a replacement incarnation" + }, + "source_executor_requirements": { + "raw_case_encoding": "each raw case has exactly one raw_body_utf8 or lowercase raw_body_hex input; hex cases bind raw_body_bytes; each decoded raw vector is at most 512 bytes", + "kinds": [ + "workflow", + "activity", + "query", + "update_validation" + ], + "inputs": "synthetic authorized namespace, registration incarnations, tasks, clocks and faults; no real credentials", + "wire_vectors": "bind every decoded and raw wire case ID to actual SDK selected-acceptance handling before payload observers/hydration; enforce strict duplicate-name rejection on the original response before ordinary lossy decoding", + "authority_vectors": "bind every authority case ID to actual native receipt/claim/registration transaction boundaries on each required backend", + "race_controls": "independent connections with barriers at actual authority locks, commits, rollback and replacement; a scenario-only model is insufficient", + "before_after_observations": [ + "logical poll and selected packet identity", + "current original/replacement incarnation and fence", + "receipt task/kind/owner/attempt", + "current lease authority including renewal", + "attempt counter", + "task-start history", + "unrelated pending logical polls" + ], + "receipt_precedence": "serialize before a negative decision; durable live receipt replays same current claim after offer expiry and restart; fenced/ended claim never replays", + "no_result_inference": "enumerating or validating this source data proves no native transaction, SDK execution, installed capability or runtime result" + }, + "wire_cases": [ + { + "id": "refusal-expiry-exact", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-authority-loss-exact", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-expiry-matching-exception", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": "task_offer_expired" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-authority-matching-exception", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-expiry-matching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409 + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-expiry-matching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": "task_offer_expired" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-expiry-mismatching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 500 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-mismatching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": "unrelated_terminal_reason" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": true + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": "409" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-200", + "input": { + "http_status": 200, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-400", + "input": { + "http_status": 400, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-401", + "input": { + "http_status": 401, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-403", + "input": { + "http_status": 403, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-404", + "input": { + "http_status": 404, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-405", + "input": { + "http_status": 405, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-408", + "input": { + "http_status": 408, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-410", + "input": { + "http_status": 410, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-422", + "input": { + "http_status": 422, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-429", + "input": { + "http_status": 429, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-500", + "input": { + "http_status": 500, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-503", + "input": { + "http_status": 503, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-task", + "input": { + "http_status": 409, + "decoded_body": { + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-poll-status", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-offer-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_offer": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-offer-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-retryable-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "retryable": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-retry-after-ms-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "retry_after_ms": 0 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-id-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_id": "synthetic-task-A" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-result-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "result": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-cross-pair", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 500, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": "409", + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": true, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-matching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409 + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-authority-matching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-authority-mismatching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 500 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-mismatching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": "unrelated_terminal_reason" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": true + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": "409" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-200", + "input": { + "http_status": 200, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-400", + "input": { + "http_status": 400, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-401", + "input": { + "http_status": 401, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-403", + "input": { + "http_status": 403, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-404", + "input": { + "http_status": 404, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-405", + "input": { + "http_status": 405, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-408", + "input": { + "http_status": 408, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-410", + "input": { + "http_status": 410, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-422", + "input": { + "http_status": 422, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-429", + "input": { + "http_status": 429, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-500", + "input": { + "http_status": 500, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-503", + "input": { + "http_status": 503, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-task", + "input": { + "http_status": 409, + "decoded_body": { + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-poll-status", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-offer-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_offer": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-offer-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-retryable-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "retryable": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-retry-after-ms-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "retry_after_ms": 0 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-id-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_id": "synthetic-task-A" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-result-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "result": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-cross-pair", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 500, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": "409", + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": true, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-string", + "input": { + "http_status": "409", + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-boolean", + "input": { + "http_status": true, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-null", + "input": { + "http_status": null, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-fraction", + "input": { + "http_status": 409.5, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": null, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": false, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": 409, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": [], + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": {}, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": 409 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": [] + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": {} + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": false, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": 0, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": "", + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": [], + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": {}, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-payload", + "input": { + "http_status": 409, + "decoded_body": { + "task": { + "task_id": "synthetic-task-A" + }, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-null", + "input": { + "http_status": 409, + "decoded_body": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-array", + "input": { + "http_status": 409, + "decoded_body": [] + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-boolean", + "input": { + "http_status": 409, + "decoded_body": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-string", + "input": { + "http_status": 409, + "decoded_body": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-integer", + "input": { + "http_status": 409, + "decoded_body": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-draining", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "draining", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stopped", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stopped", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-heartbeat-underscore-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "heartbeat_stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-heartbeat-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "heartbeat-stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale-underscore-worker-underscore-registration", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "stale_worker_registration" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-empty-poll", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "empty" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-offered-response", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "offered", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-leading-space", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": " task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-case-change", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "TASK_OFFER_EXPIRED", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-unreadable-body", + "input": { + "http_status": 409, + "decoded_body": null, + "transport_failure": "body_read_failed" + }, + "expected_action": "retain_selected_packet" + } + ], + "raw_wire_cases": [ + { + "id": "raw-refusal-expiry-exact", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-authority-exact", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "end_original_registration" + }, + { + "id": "raw-refusal-unique-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"re\\u0061son\":\"task_offer_expired\"}" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-expiry-duplicate-task-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-task-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":{\"task_id\":\"synthetic-task-A\"},\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-poll-status-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-poll-status-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"draining\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-reason-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-reason-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"worker_registration_lost_authority\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"re\\u0061son\":\"worker_registration_lost_authority\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-task-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-task-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":{\"task_id\":\"synthetic-task-A\"},\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-poll-status-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"stale_worker_registration\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-poll-status-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"draining\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-reason-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"worker_registration_lost_authority\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-reason-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"re\\u0061son\":\"task_offer_expired\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-truncated-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "invalid_json" + }, + { + "id": "raw-refusal-expiry-trailing-json-whitespace", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} \r\n\t" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-expiry-trailing-text", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} trailing" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-trailing-second-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} {}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-trailing-second-scalar", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} null" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-utf8-invalid-leading", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564ff227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-lone-continuation", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f6578706972656480227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-surrogate", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564eda080227d", + "raw_body_bytes": 81 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-truncated-multibyte", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564e2227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-trailing-json-whitespace", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} \r\n\t" + }, + "expected_action": "end_original_registration" + }, + { + "id": "raw-refusal-authority-trailing-text", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} trailing" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-trailing-second-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} {}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-trailing-second-scalar", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} null" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-utf8-invalid-leading", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479ff227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-lone-continuation", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f7269747980227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-surrogate", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479eda080227d", + "raw_body_bytes": 104 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-truncated-multibyte", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479e2227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + } + ], + "authority_cases": [ + { + "id": "authority-expired-never-accepted", + "before": "original registration current; selected offer is past deadline; no claim or receipt", + "ordered_actions": [ + "take logical-poll/claim authority locks", + "read authoritative receipt and task in serialized state", + "confirm no acceptance can commit for the selected packet", + "emit expiry refusal" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "new_claims": 0, + "new_attempts": 0, + "new_start_events": 0, + "registration_preserved": true + } + }, + { + "id": "authority-acceptance-commits-before-expiry-reader", + "before": "valid acceptance owns authority locks before deadline; expiry reader is concurrent", + "ordered_actions": [ + "acceptance commits receipt, live lease, attempt and start event atomically", + "expiry reader waits for the same authority locks", + "advance past offer deadline", + "expiry reader observes committed live receipt", + "retry original selected packet" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true, + "expiry_refusal_forbidden": true + } + }, + { + "id": "authority-expiry-decision-wins-before-late-acceptance", + "before": "past deadline; no committed receipt; independent acceptance is concurrent", + "ordered_actions": [ + "expiry reader acquires shared transition authority first", + "read authoritative receipt/task and decide expiry under the locks", + "release transaction", + "late acceptance rechecks deadline and authoritative state under the same locks" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "total_new_claims": 0, + "total_new_attempts": 0, + "total_new_start_events": 0, + "acceptance_cannot_commit_after_refusal": true + } + }, + { + "id": "authority-inflight-acceptance-rolls-back", + "before": "acceptance has uncommitted receipt/lease/attempt/start writes; expiry reader waits", + "ordered_actions": [ + "roll back all acceptance writes", + "expiry reader acquires authority locks after rollback", + "observe no committed receipt and expired original packet" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "total_new_claims": 0, + "total_new_attempts": 0, + "total_new_start_events": 0 + } + }, + { + "id": "authority-commit-ack-lost", + "before": "acceptance transaction commits but caller receives a commit/transport error", + "ordered_actions": [ + "retain selected packet on uncertain commit acknowledgement", + "advance past offer deadline", + "retry identical packet through another connection", + "serialize and read authoritative committed receipt/task" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true, + "legacy_claim_requests": 0 + } + }, + { + "id": "authority-lagging-replica-misses-receipt", + "before": "primary has a committed live claim; an older replica cannot see its receipt", + "ordered_actions": [ + "route retry past deadline to lagging replica", + "refuse to infer absence from stale state", + "retain selected packet until authoritative serialized read is possible" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "expiry_refusal_forbidden": true, + "payload_execution_authorized": false + } + }, + { + "id": "authority-receipt-read-fails", + "before": "receipt/task authority read is unavailable or transaction outcome is unknown", + "ordered_actions": [ + "fail authoritative read or serialization", + "do not manufacture a no-claim proof from timeout/error/empty response" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "definitive_refusal_forbidden": true, + "payload_execution_authorized": false + } + }, + { + "id": "authority-replay-after-process-restart", + "before": "acceptance committed a live claim and durable receipt before process exit", + "ordered_actions": [ + "restart accepting process", + "advance past offer deadline while lease remains live", + "retry original packet on a fresh capable process", + "read durable receipt and current authority" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true + } + }, + { + "id": "authority-renewed-live-claim", + "before": "committed claim was renewed beyond its original lease and offer deadline", + "ordered_actions": [ + "advance beyond original lease expiry but before current renewed lease expiry", + "retry identical selected packet", + "read receipt and current lease rather than original token deadline" + ], + "expected": { + "outcome": "replay_original_live_claim", + "same_task_owner_attempt": true, + "retry_new_attempts": 0, + "retry_new_start_events": 0 + } + }, + { + "id": "authority-live-receipt-premature-prune", + "before": "a live replayable claim exists but its receipt has incorrectly disappeared", + "ordered_actions": [ + "detect inability to establish required durable live-claim receipt invariant", + "do not translate missing receipt into an expired-unaccepted claim" + ], + "expected": { + "outcome": "invariant_failure_uncertain", + "selected_packet_preserved": true, + "expiry_refusal_forbidden": true, + "implementation_qualification_must_fail": true + } + }, + { + "id": "authority-draining-retains-live-claim", + "before": "original registration and committed claim remain authoritative; worker mode becomes draining", + "ordered_actions": [ + "retry selected packet past offer deadline", + "read committed live receipt before considering worker mode" + ], + "expected": { + "outcome": "replay_original_live_claim", + "same_task_owner_attempt": true, + "mode_alone_cannot_erase_receipt": true, + "authority_loss_refusal_forbidden": true, + "retry_new_attempts": 0, + "retry_new_start_events": 0 + } + }, + { + "id": "authority-draining-no-proof", + "before": "only a draining lifecycle label is known; actual original claim/registration authority has not been proved lost", + "ordered_actions": [ + "receive a draining label while an acceptance is selected", + "preserve original packet and await one of the exact proved outcomes" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false, + "payload_execution_authorized": false + } + }, + { + "id": "authority-stopped-label-no-proof", + "before": "only a stopped lifecycle label is known; no authoritative ended-identity proof", + "ordered_actions": [ + "inspect lifecycle label without replacing authoritative receipt/registration read" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-heartbeat-label-no-proof", + "before": "heartbeat is stale or freshness lookup is unavailable; authority revocation is unproved", + "ordered_actions": [ + "do not equate heartbeat age with an authoritative execution fence" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-replaced-registration-fenced", + "before": "replacement incarnation has authoritatively fenced original registration and all its execution authority", + "ordered_actions": [ + "serialize replacement and original-packet acceptance under normal transition authority", + "read original identity/fence and prove original worker can no longer execute", + "emit authority-loss refusal" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0, + "replacement_authority_unchanged": true + } + }, + { + "id": "authority-ended-registration-proved", + "before": "original registration ended and authoritative task/lease state proves it cannot execute", + "ordered_actions": [ + "read original registration identity plus current claim/lease authority under transition locks", + "prove actual authority loss rather than infer it from mode text" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0 + } + }, + { + "id": "authority-same-second-replacement", + "before": "same worker ID and process metadata recur within one stored timestamp second; original incarnation differs", + "ordered_actions": [ + "use durable incarnation/credential/fence identity rather than timestamp or worker ID alone", + "prove original execution authority revoked" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0, + "replacement_authority_unchanged": true + } + }, + { + "id": "authority-registration-row-unavailable", + "before": "original registration lookup is missing through incomplete/failed observation, not proved authoritative absence", + "ordered_actions": [ + "refuse authority-loss conclusion without a complete authoritative identity/fence read" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-selected-claim-ended-registration-current", + "before": "selected claim has ended or its task attempt changed, but original registration still has execution authority for other work", + "ordered_actions": [ + "forbid replay of ended/replaced selected claim", + "do not infer registration-wide authority loss from this task alone", + "retain original packet without inventing a third definitive refusal pair" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "old_payload_replayed": false, + "end_original_registration": false, + "other_claim_authority_unchanged": true + } + }, + { + "id": "authority-replacement-wins-race", + "before": "replacement and acceptance race on independent connections for the original incarnation", + "ordered_actions": [ + "replacement acquires authority locks first", + "replacement commits fence/ended original execution authority", + "old acceptance then reads current registration and task authority" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "total_new_claims_from_old_packet": 0, + "total_new_attempts_from_old_packet": 0, + "total_new_start_events_from_old_packet": 0 + } + }, + { + "id": "authority-acceptance-wins-then-replacement", + "before": "valid acceptance and replacement race on independent connections", + "ordered_actions": [ + "acceptance acquires authority locks first and atomically commits one claim/receipt/attempt/start", + "replacement then serializes and revokes original execution authority", + "retry original selected packet after replacement", + "read receipt but also current authority; never replay a fenced claim" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "old_payload_replayed_after_fence": false + } + }, + { + "id": "authority-expired-offer-live-registration-other-claim", + "before": "selected offer never accepted and is expired; task is now leased by a peer; original registration remains current", + "ordered_actions": [ + "serialize receipt/task read and prove no selected-packet acceptance can commit", + "preserve peer lease and original registration" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "new_claims_from_selected_packet": 0, + "peer_authority_unchanged": true, + "registration_preserved": true + } + }, + { + "id": "authority-refusal-preserves-unrelated-polls", + "before": "two distinct logical polls are pending on the same registration; only selected poll A is proved expired", + "ordered_actions": [ + "receive exact expiry refusal for A", + "resolve only A", + "preserve pending original packet and identity for B" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "poll_b_packet_unchanged": true, + "registration_preserved": true + } + }, + { + "id": "authority-inconsistent-envelope-preserves-all", + "before": "selected poll A and unrelated B are pending; transport/exception/body disagree", + "ordered_actions": [ + "receive mixed or extra-key refusal candidate", + "preserve A and B", + "suppress payload observer, hydration and execution", + "do not begin a replacement logical poll or legacy claim" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "poll_b_packet_unchanged": true, + "end_original_registration": false, + "new_logical_poll_requests": 0, + "legacy_claim_requests": 0 + } + } + ] +} From 3b55e5dbf67a5b421f680ed6b05b5f0de955ebce Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:18:29 +0000 Subject: [PATCH 08/12] Split JSON scalar scanning and preserve valid claim values --- src/Transport/UniqueJson.php | 7 ++++++- tests/WorkerTaskDeliveryWireTest.php | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Transport/UniqueJson.php b/src/Transport/UniqueJson.php index a8c7611f..19851b71 100644 --- a/src/Transport/UniqueJson.php +++ b/src/Transport/UniqueJson.php @@ -37,8 +37,13 @@ private static function value(string $json, int &$offset, int $depth): void self::items($json, $offset, $depth + 1); } elseif ($token === '"') { self::stringToken($json, $offset); - } elseif (preg_match('/\G(?:true|false|null|-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)/A', $json, $match, 0, $offset) === 1) { + } elseif (preg_match('/\G(?:true|false|null)/A', $json, $match, 0, $offset) === 1) { $offset += strlen($match[0]); + } elseif (preg_match('/\G-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?/A', $json, $match, 0, $offset) === 1) { + $offset += strlen($match[0]); + if (preg_match('/\G[eE][+-]?[0-9]+/A', $json, $exponent, 0, $offset) === 1) { + $offset += strlen($exponent[0]); + } } else { throw new JsonException('Invalid JSON value.'); } diff --git a/tests/WorkerTaskDeliveryWireTest.php b/tests/WorkerTaskDeliveryWireTest.php index a465ff96..3669f9c1 100644 --- a/tests/WorkerTaskDeliveryWireTest.php +++ b/tests/WorkerTaskDeliveryWireTest.php @@ -68,6 +68,13 @@ public function testTheOriginalWireReplyControlsOnlyItsSelectedAcceptance(array $error = null; try { $result = self::poll($client, $endpoint, 'original'); } catch (ServerException $caught) { $error = $caught; } + if ($action === 'deliver_selected_task') { + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame($input['expected_claim'], $result); + self::assertSame(1, $observed); + self::assertCount(3, $requests); + return; + } self::assertSame(0, $observed, 'No refusal or uncertain outcome may start task execution.'); if ($action === 'retain_selected_packet') { self::assertInstanceOf(TaskDeliveryUncertain::class, $error); @@ -97,6 +104,22 @@ public static function wireCases(): iterable } } } + $payload = <<<'JSON' +{"zero":0,"negative":-3,"decimal":0.25,"tiny":1e-3,"huge":2E+4,"bool":true,"nil":null,"nested":[false,{},[]],"unicode":"\uD83D\uDE80","quotes":"a\"b","slash":"a\\b","names":{"re\u0061son":1,"other":2}} +JSON; + $expected = ['zero' => 0, 'negative' => -3, 'decimal' => 0.25, 'tiny' => 0.001, 'huge' => 20000.0, + 'bool' => true, 'nil' => null, 'nested' => [false, [], []], 'unicode' => '🚀', 'quotes' => 'a"b', + 'slash' => 'a\\b', 'names' => ['reason' => 1, 'other' => 2]]; + foreach ([['workflow', 'workflow'], ['workflow', 'update_validation'], ['activity', 'activity'], ['query', 'query'], ['update-validation', 'update_validation']] as [$endpoint, $kind]) { + $claim = self::claimed($kind); + $raw = substr(json_encode($claim, JSON_THROW_ON_ERROR), 0, -2).',"custom_data":'.$payload.'}}'; + $claim['task']['custom_data'] = $expected; + foreach ([false, true] as $progress) { + yield 'valid claim values / '.$endpoint.' / '.$kind.($progress ? ' / progress' : '') => [ + ['http_status' => 200, 'raw_body_utf8' => $raw, 'expected_claim' => $claim], 'deliver_selected_task', $endpoint, $kind, true, $progress, + ]; + } + } } /** @param array $input */ From e5bf977e39fec198f5f7e11e691902dd699056cf Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:00:41 +0000 Subject: [PATCH 09/12] Rebind task delivery replies to the current registration [skip ci] --- src/Client.php | 6 +- src/Worker/TaskDelivery.php | 14 ++ tests/WorkerTaskDeliveryRegistrationTest.php | 144 +++++++++++++++++++ 3 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 tests/WorkerTaskDeliveryRegistrationTest.php diff --git a/src/Client.php b/src/Client.php index e4400dfb..de0f6a7b 100644 --- a/src/Client.php +++ b/src/Client.php @@ -2808,8 +2808,10 @@ private function performPollTaskResponse(string $path, array $body): array return $this->taskDelivery->poll($path, $body, fn (string $wirePath, array $wireBody, ?TaskDeliveryAttempt $selected): array => $this->request('POST', $wirePath, true, $wireBody, - responseValidator: static fn (array $response) => - TaskDelivery::assertResponse($response, $wireBody, $wirePath, $selected))) + responseValidator: function (array $response) use ($wireBody, $wirePath, $selected): void { + $this->taskDelivery->assertCurrentRegistration($wireBody); + TaskDelivery::assertResponse($response, $wireBody, $wirePath, $selected); + })) ?? $this->worker('POST', $path, $body); } catch (ServerException $exception) { if ($retryTransientFailure && $exception->isTransientConnectionFailure()) { diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php index a845e790..606dd6e6 100644 --- a/src/Worker/TaskDelivery.php +++ b/src/Worker/TaskDelivery.php @@ -107,6 +107,7 @@ private function deliver(string $key, string $wirePath, array $body, array $allo try { if ($selected === null) { $response = $send($wirePath, $body, null); + $this->assertCurrentRegistration($body); if (!array_key_exists('task_offer', $response)) { return $response; } @@ -114,6 +115,7 @@ private function deliver(string $key, string $wirePath, array $body, array $allo $selected = $this->pending[$key] ??= $candidate; } $response = $send($selected->path, $selected->requestBody(), $selected); + $this->assertCurrentRegistration($selected->requestBody()); if ($response['task'] === null) { throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); } @@ -131,6 +133,18 @@ private function deliver(string $key, string $wirePath, array $body, array $allo return $response; } + /** @internal Recheck after I/O and before observing or returning its response. + * @param array $body + */ + public function assertCurrentRegistration(array $body): void + { + $current = $this->registrations[(string) $body['worker_id']] ?? null; + if ($current === null || $current !== ['queue' => $body['task_queue'] ?? null, 'token' => $body['registration_token'] ?? null]) { + throw new ServerException('The task-delivery registration changed before its response.', 200, + 'stale_task_delivery_registration'); + } + } + /** * Validate before payload hydration can initialize a claim supervisor. * @param array $response diff --git a/tests/WorkerTaskDeliveryRegistrationTest.php b/tests/WorkerTaskDeliveryRegistrationTest.php new file mode 100644 index 00000000..e2b34ff1 --- /dev/null +++ b/tests/WorkerTaskDeliveryRegistrationTest.php @@ -0,0 +1,144 @@ + 1 && in_array($change, ['token', 'unrelated'], true) ? 'b' : 'a'); + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { + if ($poll === 'original' && $phase === 'offer') { Fiber::suspend(); } + return self::offer($kind, $poll); + } + if ($poll === 'original' && $phase === 'accept') { Fiber::suspend(); } + if ($poll === 'replacement' && ++$replacementAttempts === 1) { + throw new TransportException('Synthetic lost response.', 504); + } + return self::claimed($kind, $poll); + }); + $observed = 0; + $client = (new Client('https://server.example', transport: $transport, + metrics: $metrics ? new RecordingMetricsRecorder() : null))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed, $phase): ?Closure { + if (($body['poll_request_id'] ?? null) === 'original' && ($response['task'] ?? null) !== null) { + ++$observed; + if ($phase === 'observer') { Fiber::suspend(); } + } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + $old = new Fiber(static function () use ($client, $endpoint, $kind): array { + try { return ['result' => self::poll($client, $endpoint, $kind, 'original')]; } + catch (ServerException $error) { return ['error' => $error]; } + }); + $old->start(); + self::assertTrue($old->isSuspended()); + $replacement = clone $client; + $queue = $change === 'queue' ? 'other-orders' : 'orders'; + $replacement->registerWorker($change === 'unrelated' ? 'other-worker' : 'worker', $queue, [], []); + try { + self::poll($replacement, $endpoint, $kind, 'replacement', $queue); + self::fail('The replacement fixture must retain its uncertain packet.'); + } catch (TaskDeliveryUncertain) { + } + $old->resume(); + $outcome = $old->getReturn(); + if (in_array($change, ['same', 'unrelated'], true)) { + self::assertSame(self::claimed($kind, 'original'), $outcome['result'] ?? null); + self::assertSame(1, $observed); + } else { + self::assertArrayNotHasKey('result', $outcome, 'A stale successful receipt cannot authorize execution.'); + self::assertInstanceOf(ServerException::class, $outcome['error'] ?? null); + self::assertSame($phase === 'observer' ? 1 : 0, $observed, 'Transport replacement must refuse before payload observation.'); + } + self::assertSame(self::claimed($kind, 'replacement'), self::poll($replacement, $endpoint, $kind, 'replacement', $queue)); + $packets = array_values(array_filter($transport->requests, static fn (array $request): bool => + ($request['body']['poll_request_id'] ?? null) === 'replacement' && isset($request['body']['task_offer_token']))); + self::assertCount(2, $packets); + self::assertSame($packets[0], $packets[1], 'The old response must not mutate or erase the replacement packet.'); + self::assertSame(str_repeat($change === 'token' ? 'b' : 'a', 32), $packets[1]['body']['registration_token']); + self::assertSame($queue, $packets[1]['body']['task_queue']); + } + + /** @return iterable */ + public static function registrationRaces(): iterable + { + $cases = [ + ['workflow', 'workflow', 'accept', 'token'], + ['workflow', 'update_validation', 'accept', 'token'], + ['activity', 'activity', 'accept', 'token'], + ['query', 'query', 'accept', 'token'], + ['update-validation', 'update_validation', 'accept', 'token'], + ['workflow', 'workflow', 'offer', 'token'], + ['workflow', 'workflow', 'accept', 'queue'], + ['workflow', 'workflow', 'offer', 'queue'], + ['workflow', 'workflow', 'observer', 'token'], + ['workflow', 'workflow', 'accept', 'same'], + ['workflow', 'workflow', 'accept', 'unrelated'], + ]; + foreach ($cases as $case) { + foreach ([false, true] as $metrics) { + yield implode(' / ', $case).($metrics ? ' / metrics' : '') => [...$case, $metrics]; + } + } + } + + /** @return array */ + private static function poll(Client $client, string $endpoint, string $kind, string $poll, string $queue = 'orders'): array + { + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => $queue, 'timeout_seconds' => 4, 'poll_request_id' => $poll, 'task_kinds' => [$kind], + ]); + } + + /** @return array */ + private static function registration(string $token): array + { + return ['registered' => true, 'registration_token' => str_repeat($token, 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, 'version' => TaskDelivery::VERSION, + 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind, string $poll): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => $poll, 'token' => 'synthetic-'.$poll, 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind, string $poll): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => $poll, 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } +} From 5807c37ff5e7acaf595759dcb0a3cb78a64ea1c0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:59:29 +0000 Subject: [PATCH 10/12] Clarify numeric JSON and replacement-packet assertions --- src/Transport/UniqueJson.php | 4 +-- tests/UniqueJsonNumberTest.php | 36 ++++++++++++++++++++ tests/WorkerTaskDeliveryRegistrationTest.php | 4 ++- 3 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 tests/UniqueJsonNumberTest.php diff --git a/src/Transport/UniqueJson.php b/src/Transport/UniqueJson.php index 19851b71..a15cf743 100644 --- a/src/Transport/UniqueJson.php +++ b/src/Transport/UniqueJson.php @@ -39,9 +39,9 @@ private static function value(string $json, int &$offset, int $depth): void self::stringToken($json, $offset); } elseif (preg_match('/\G(?:true|false|null)/A', $json, $match, 0, $offset) === 1) { $offset += strlen($match[0]); - } elseif (preg_match('/\G-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?/A', $json, $match, 0, $offset) === 1) { + } elseif (preg_match('/\G-?(?:0|[1-9]\d*)(?:\.\d+)?/A', $json, $match, 0, $offset) === 1) { $offset += strlen($match[0]); - if (preg_match('/\G[eE][+-]?[0-9]+/A', $json, $exponent, 0, $offset) === 1) { + if (preg_match('/\G[eE][+-]?\d+/A', $json, $exponent, 0, $offset) === 1) { $offset += strlen($exponent[0]); } } else { diff --git a/tests/UniqueJsonNumberTest.php b/tests/UniqueJsonNumberTest.php new file mode 100644 index 00000000..ffbfd315 --- /dev/null +++ b/tests/UniqueJsonNumberTest.php @@ -0,0 +1,36 @@ +expectException(JsonException::class); + UniqueJson::decode($json); + } + + /** @return iterable */ + public static function invalidNumbers(): iterable + { + foreach (['١', '１', '1١', '1.١', '1e١', '-١', '1e+１', '[1,١]', '{"n":1.१}', + '01', '1.', '1e', '1e+', 'NaN', 'Infinity'] as $json) { + yield $json => [$json]; + } + } +} diff --git a/tests/WorkerTaskDeliveryRegistrationTest.php b/tests/WorkerTaskDeliveryRegistrationTest.php index e2b34ff1..1a4c0318 100644 --- a/tests/WorkerTaskDeliveryRegistrationTest.php +++ b/tests/WorkerTaskDeliveryRegistrationTest.php @@ -77,7 +77,9 @@ static function (string $path, ?array $body, array $response) use (&$observed, $ self::assertInstanceOf(ServerException::class, $outcome['error'] ?? null); self::assertSame($phase === 'observer' ? 1 : 0, $observed, 'Transport replacement must refuse before payload observation.'); } - self::assertSame(self::claimed($kind, 'replacement'), self::poll($replacement, $endpoint, $kind, 'replacement', $queue)); + $expected = self::claimed($kind, 'replacement'); + $actual = self::poll($replacement, $endpoint, $kind, 'replacement', $queue); + self::assertSame($expected, $actual); $packets = array_values(array_filter($transport->requests, static fn (array $request): bool => ($request['body']['poll_request_id'] ?? null) === 'replacement' && isset($request['body']['task_offer_token']))); self::assertCount(2, $packets); From 072b3d34e3842c5443f0637eeff5c46e19fb452d Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:02:44 +0000 Subject: [PATCH 11/12] Name the expected observer count in registration races --- tests/WorkerTaskDeliveryRegistrationTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/WorkerTaskDeliveryRegistrationTest.php b/tests/WorkerTaskDeliveryRegistrationTest.php index 1a4c0318..ff2c0b31 100644 --- a/tests/WorkerTaskDeliveryRegistrationTest.php +++ b/tests/WorkerTaskDeliveryRegistrationTest.php @@ -75,7 +75,8 @@ static function (string $path, ?array $body, array $response) use (&$observed, $ } else { self::assertArrayNotHasKey('result', $outcome, 'A stale successful receipt cannot authorize execution.'); self::assertInstanceOf(ServerException::class, $outcome['error'] ?? null); - self::assertSame($phase === 'observer' ? 1 : 0, $observed, 'Transport replacement must refuse before payload observation.'); + $expectedObservations = $phase === 'observer' ? 1 : 0; + self::assertSame($expectedObservations, $observed, 'Transport replacement must refuse before payload observation.'); } $expected = self::claimed($kind, 'replacement'); $actual = self::poll($replacement, $endpoint, $kind, 'replacement', $queue); From 8bc1ba0cda6c046a2e470b37c16c4fb682c8a339 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:10:15 +0000 Subject: [PATCH 12/12] Clarify callback counts for each registration race phase --- tests/WorkerTaskDeliveryRegistrationTest.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/WorkerTaskDeliveryRegistrationTest.php b/tests/WorkerTaskDeliveryRegistrationTest.php index ff2c0b31..5e0acb88 100644 --- a/tests/WorkerTaskDeliveryRegistrationTest.php +++ b/tests/WorkerTaskDeliveryRegistrationTest.php @@ -75,8 +75,11 @@ static function (string $path, ?array $body, array $response) use (&$observed, $ } else { self::assertArrayNotHasKey('result', $outcome, 'A stale successful receipt cannot authorize execution.'); self::assertInstanceOf(ServerException::class, $outcome['error'] ?? null); - $expectedObservations = $phase === 'observer' ? 1 : 0; - self::assertSame($expectedObservations, $observed, 'Transport replacement must refuse before payload observation.'); + if ($phase === 'observer') { + self::assertSame(1, $observed, 'The callback had already observed the payload before registration changed.'); + } else { + self::assertSame(0, $observed, 'Transport replacement must refuse before payload observation.'); + } } $expected = self::claimed($kind, 'replacement'); $actual = self::poll($replacement, $endpoint, $kind, 'replacement', $queue);