diff --git a/docs/portal/build/workers.md b/docs/portal/build/workers.md index 41e9e64b..8544c1d9 100644 --- a/docs/portal/build/workers.md +++ b/docs/portal/build/workers.md @@ -99,6 +99,17 @@ An activity option such as `['queue' => 'payments-v2']` selects the queue of a n ### Poll timeouts and early disconnects +When the Server advertises `task_delivery`, the managed worker negotiates +offer/accept delivery. Receiving an offer does not reserve a task or start an +activity attempt. The SDK accepts the selected offer before running a handler. +If that response is lost, it retries the same packet and poll identity until +the Server confirms the claim, expiry, or loss of registration authority. +An older node returning an error cannot switch that poll to legacy delivery. +Servers without this capability continue using the existing poll protocol. +During acceptance, stop/drain hints and malformed replies retain that packet. +Only the protocol's two closed HTTP `409` refusal shapes can release it or end +the original registration; a later replacement registration is preserved. + With the default cURL transport, supporting Servers send progress during idle worker polls. Progress keeps an idle connection active while the complete request remains bounded to the requested wait plus five seconds. Ordinary polls diff --git a/docs/protocol.md b/docs/protocol.md index 6ce822cb..7c9714cf 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -198,6 +198,32 @@ may have committed a claim before the connection disappeared, so `task=null` does not establish that no lease exists. This does not make arbitrary database errors or unrelated structured HTTP 500/503 responses retryable. +With negotiated `task_delivery`, worker polls use `/offers`. An offer alone +grants no execution authority. Positive acceptance includes the original +registration token and selected offer packet; its request body remains fixed +through uncertain replies. Managed workers retain this state across client +clones and honor the normal retry backoff. If using the low-level +`poll*TaskResponse()` methods, catch `TaskDeliveryUncertain` and retry with its +`details['poll_request_id']` on the same Client. An ordinary empty response to +an acceptance does not prove that no claim committed. Definitive refusal requires +HTTP `409` and exactly `task`, `poll_status`, and `reason`, with literal +`task: null`. The only accepted status/reason pairs are +`task_offer_expired` / `task_offer_expired` and +`stale_worker_registration` / `worker_registration_lost_authority`. +Expiry releases only the selected packet; authority loss ends only its original +registration incarnation. Other pending requests and replacement registrations +remain separate. The Server must establish these outcomes from receipt-first +reconciliation and authoritative claim/registration state. + +The default transport checks the complete original UTF-8 JSON response for +duplicate decoded member names before associative decoding. This also applies +inside a progress envelope. Invalid bytes, trailing data, extra fields, mixed +pairs, conflicting exception fields and successful terminal hints retain the +selected packet without running a handler. Discovery before selection keeps its +existing poll classification. A custom transport must provide original-byte +validation through `Transport\TaskDeliveryRefusal::fromJson()` and attach that +evidence to `TransportException`; a decoded-only refusal remains uncertain. + Before replaying or completing a workflow task, the managed worker requires a successful lease-renewal response whose task ID, attempt, and lease owner match the current claim. A typed `renewed=false`, `retryable=true` response is retried diff --git a/src/Client.php b/src/Client.php index 6ab2b034..de0f6a7b 100644 --- a/src/Client.php +++ b/src/Client.php @@ -68,6 +68,9 @@ use DurableWorkflow\Worker\WorkerSessionOptions; use DurableWorkflow\Worker\ActivityLeaseBinding; use DurableWorkflow\Worker\ActivityLeaseProcess; +use DurableWorkflow\Worker\TaskDelivery; +use DurableWorkflow\Worker\TaskDeliveryAttempt; +use DurableWorkflow\Transport\PollTransport; use DurableWorkflow\Exception\PaginationException; use InvalidArgumentException; @@ -88,6 +91,7 @@ final class Client implements WorkflowClientInterface private FailureReportLimiter $failureReports; private bool $boundedWorkerRequests = false; private ActivityLeaseBinding $activityLeaseBinding; + private TaskDelivery $taskDelivery; private ?string $activityLeaseBootstrap = null; private ?ExecutionTelemetry $executionTelemetry = null; private bool $observabilityContextSupported = false; @@ -162,6 +166,7 @@ public function __construct( $this->payloadUploads = new RuntimePayloadUploads($this->transport, $this->baseUri); $this->failureReports = new FailureReportLimiter(new FailureReportLimits(), $this->transport); $this->activityLeaseBinding = new ActivityLeaseBinding(); + $this->taskDelivery = new TaskDelivery(); $this->metricsObserver = new MetricsObserver($metrics ?? new NoopMetricsRecorder()); } @@ -1752,6 +1757,7 @@ public function registerWorker( 'memo_upserts', 'typed_search_attributes', 'durable_selection', + TaskDelivery::CAPABILITY, ], int $maxConcurrentWorkflowTasks = 1, int $maxConcurrentActivityTasks = 1, @@ -1761,6 +1767,10 @@ public function registerWorker( int $maxConcurrentWorkerSessions = 10, ?array $workflowDefinitionFingerprints = null, ): array { + if ($this->workerProtocolVersion !== '1.20') { + $capabilities = array_values(array_filter($capabilities, + static fn (string $capability): bool => $capability !== TaskDelivery::CAPABILITY)); + } if (in_array('message_streams', $capabilities, true) && !Version::supportsMessageStreams()) { throw new InvalidArgumentException('Message streams require worker protocol 1.15 or newer.'); } @@ -1797,7 +1807,9 @@ public function registerWorker( 'max_concurrent_activity_tasks' => $maxConcurrentActivityTasks, 'max_concurrent_worker_sessions' => $maxConcurrentWorkerSessions, 'build_id' => $buildId, + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ])); + $this->taskDelivery->registered($workerId, $taskQueue, $this->workerProtocolVersion, $capabilities, $registration); if (array_key_exists('server_capabilities', $registration)) { $this->rememberClaimedTaskReadyAge($registration['server_capabilities']); } @@ -1859,6 +1871,7 @@ public function heartbeatWorker(string $workerId, array $taskSlots = [], ?Reques public function deregisterWorker(string $workerId): void { $this->control('DELETE', '/workers/'.$this->segment($workerId)); + $this->taskDelivery->forget($workerId); } /** @return array{worker_id: string, outcome: string, recovered_workflow_task_count: int} */ @@ -1883,6 +1896,8 @@ public function deregisterWorkerRegistration(string $workerId, ?string $registra 'invalid_worker_deregistration_receipt', $response); } + $this->taskDelivery->forget($workerId); + /** @var array{worker_id: string, outcome: string, recovered_workflow_task_count: int} $response */ return $response; } @@ -2790,7 +2805,14 @@ private function performPollTaskResponse(string $path, array $body): array $retryTransientFailure = true; while (true) { try { - return $this->worker('POST', $path, $body); + return $this->taskDelivery->poll($path, $body, + fn (string $wirePath, array $wireBody, ?TaskDeliveryAttempt $selected): array => + $this->request('POST', $wirePath, true, $wireBody, + responseValidator: function (array $response) use ($wireBody, $wirePath, $selected): void { + $this->taskDelivery->assertCurrentRegistration($wireBody); + TaskDelivery::assertResponse($response, $wireBody, $wirePath, $selected); + })) + ?? $this->worker('POST', $path, $body); } catch (ServerException $exception) { if ($retryTransientFailure && $exception->isTransientConnectionFailure()) { $retryTransientFailure = false; @@ -2857,7 +2879,7 @@ private function sendTransportRequest(string $method, string $path, bool $worker } /** @return array */ - private function requestHeaders(bool $worker): array + private function requestHeaders(bool $worker, string $path = ''): array { $headers = [ 'Accept' => 'application/json', @@ -2869,6 +2891,9 @@ private function requestHeaders(bool $worker): array if ($this->authentication !== null) { $headers = array_merge($headers, $this->authentication->headers($worker)); } + if ($worker && PollTransport::taskKind('POST', $path) !== null && str_ends_with($path, '/offers')) { + $headers[TaskDelivery::HEADER] = TaskDelivery::VERSION; + } return $headers; } @@ -2876,6 +2901,7 @@ private function requestHeaders(bool $worker): array /** * @param array|null $body * @param (\Closure(array): list>)|null $payloadPaths + * @param (\Closure(array): void)|null $responseValidator * @return array */ private function request( @@ -2887,14 +2913,16 @@ private function request( ?RequestBudget $budget = null, ?\Closure $payloadPaths = null, ?string $requestId = null, + ?\Closure $responseValidator = null, ): array { if ($this->metricsRequest !== null || !$this->metricsObserver->enabled()) { - return $this->performRequest($method, $path, $worker, $body, $operation, $budget, $payloadPaths, $requestId, $this->metricsRequest); + return $this->performRequest($method, $path, $worker, $body, $operation, $budget, $payloadPaths, $requestId, + $this->metricsRequest, $responseValidator); } return $this->observeRequest($method, $path, $worker, fn (ClientRequestObservation $observation): array => $this->performRequest( - $method, $path, $worker, $body, $operation, $budget, $payloadPaths, $requestId, $observation)); + $method, $path, $worker, $body, $operation, $budget, $payloadPaths, $requestId, $observation, $responseValidator)); } /** @param \Closure(ClientRequestObservation): array $operation @@ -2928,14 +2956,16 @@ private function performRequest( ?\Closure $payloadPaths, ?string $requestId, ?ClientRequestObservation $observation, + ?\Closure $responseValidator, ): array { if ($observation !== null) { $observation->status = null; } - $headers = $this->requestHeaders($worker); + $headers = $this->requestHeaders($worker, $path); + $logicalPath = PollTransport::logicalPath($path); try { $callerBudget = $budget; - $poll = str_ends_with($path, '/poll') && is_int($body['timeout_seconds'] ?? null); + $poll = PollTransport::taskKind($method, $path) !== null && is_int($body['timeout_seconds'] ?? null); $pollSeconds = $poll ? max(0, min(60, $body['timeout_seconds'])) : 0; $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; if ($body !== null) { @@ -2945,7 +2975,7 @@ private function performRequest( $body = $this->failureReports->request($path, $body); } $activityLease = $worker ? $this->activityLeaseBinding->current() : null; - if ($activityLease !== null && $activityLease->governs($path)) { + if ($activityLease !== null && $activityLease->governs($logicalPath)) { // Uploading an external result is not publication authority. // Check again before its final Worker POST and bound the send // by the latest acknowledged grant, not an old cached lease. @@ -2959,21 +2989,27 @@ private function performRequest( } $budget?->remainingSeconds(); + if ($responseValidator !== null) { + if (!is_array($response) || array_is_list($response)) { + throw new ServerException('The task-delivery response is invalid.', 200, 'invalid_task_delivery_response'); + } + $responseValidator($response); + } // Control I/O and supervised payload I/O have separate bounds. // Explicit caller/cleanup budgets still cover their entire request. $responseBudget = $budget !== null && $poll && $callerBudget === null ? new RequestBudget(5) : $budget; - $canStreamBounded = $worker && ($this->boundedWorkerRequests || $path === '/worker/query-tasks/poll') + $canStreamBounded = $worker && ($this->boundedWorkerRequests || $logicalPath === '/worker/query-tasks/poll') && $this->transport instanceof \DurableWorkflow\Transport\BoundedProgressPayloadTransport && $this->transport instanceof BoundedTransport && $this->transport->supportsBoundedRequests(); $observerBudget = $canStreamBounded && $callerBudget === null ? null : $responseBudget; $progress = $worker && $this->workerPayloadObserver !== null && is_array($response) && !array_is_list($response) - ? ($this->workerPayloadObserver)($path, $body, $response, $observerBudget) : null; + ? ($this->workerPayloadObserver)($logicalPath, $body, $response, $observerBudget) : null; $streamBounded = $canStreamBounded && $progress !== null; if ($streamBounded && $callerBudget === null) { $responseBudget = null; } $result = is_array($response) && !array_is_list($response) - ? (new RuntimePayloads($this->transport, $this->baseUri, $headers, $this->maxExternalPayloadBytes, $responseBudget, $progress, $streamBounded))->response($response, $path, $worker, $payloadPaths?->__invoke($response)) + ? (new RuntimePayloads($this->transport, $this->baseUri, $headers, $this->maxExternalPayloadBytes, $responseBudget, $progress, $streamBounded))->response($response, $logicalPath, $worker, $payloadPaths?->__invoke($response)) : []; $responseBudget?->remainingSeconds(); @@ -2983,7 +3019,8 @@ private function performRequest( throw $exception; } $details = $exception->response; - $reason = is_array($details) && isset($details['reason']) ? (string) $details['reason'] : null; + $reason = is_array($details) && isset($details['reason']) + && ($responseValidator === null || is_string($details['reason'])) ? (string) $details['reason'] : null; $status = $exception->status ?? 0; $message = $exception->getMessage(); if ($operation === 'query') { diff --git a/src/Exception/TaskDeliveryUncertain.php b/src/Exception/TaskDeliveryUncertain.php new file mode 100644 index 00000000..035f351c --- /dev/null +++ b/src/Exception/TaskDeliveryUncertain.php @@ -0,0 +1,23 @@ + null, 'poll_request_id' => $pollRequestId, + 'retry_same_poll_request_id' => true, + ...($previous instanceof ServerException && is_int($previous->details['retry_after_seconds'] ?? null) + && $previous->details['retry_after_seconds'] > 0 + ? ['retry_after_seconds' => $previous->details['retry_after_seconds']] : []), + ], $previous); + } +} diff --git a/src/Exception/TransportException.php b/src/Exception/TransportException.php index c21f4640..ad7c1cb2 100644 --- a/src/Exception/TransportException.php +++ b/src/Exception/TransportException.php @@ -5,6 +5,7 @@ namespace DurableWorkflow\Exception; use Throwable; +use DurableWorkflow\Transport\TaskDeliveryRefusal; class TransportException extends DurableWorkflowException { @@ -17,6 +18,7 @@ public function __construct( public readonly bool $transientConnectionFailure = false, /** @var array */ public readonly array $headers = [], + public readonly ?TaskDeliveryRefusal $taskDeliveryRefusal = null, ) { parent::__construct($message, $status ?? 0, $previous); } diff --git a/src/Transport/PollTransport.php b/src/Transport/PollTransport.php index 4aaf2088..3f719441 100644 --- a/src/Transport/PollTransport.php +++ b/src/Transport/PollTransport.php @@ -19,7 +19,8 @@ public static function taskKind(string $method, string $uri): ?string $path = parse_url($uri, PHP_URL_PATH); if (strtoupper($method) === 'POST' && is_string($path)) { foreach (self::KINDS as $kind) { - if (str_ends_with($path, '/worker/'.$kind.'-tasks/poll')) { + if (str_ends_with($path, '/worker/'.$kind.'-tasks/poll') + || str_ends_with($path, '/worker/'.$kind.'-tasks/offers')) { return $kind; } } @@ -28,6 +29,13 @@ public static function taskKind(string $method, string $uri): ?string return null; } + /** Payload observers and claim supervision use the original logical poll. */ + public static function logicalPath(string $path): string + { + return self::taskKind('POST', $path) !== null && str_ends_with($path, '/offers') + ? substr($path, 0, -7).'/poll' : $path; + } + /** @return array{status: int, headers: array, body: array} */ public static function decode(string $rawBody, int $outerStatus): array { diff --git a/src/Transport/Psr18Transport.php b/src/Transport/Psr18Transport.php index 8cc679e2..40bc0c62 100644 --- a/src/Transport/Psr18Transport.php +++ b/src/Transport/Psr18Transport.php @@ -145,7 +145,11 @@ private function sendJson(string $method, string $uri, array $headers, ?array $b bounded: $timeoutSeconds !== null || $controlTimeout !== null, controlTimeout: $controlTimeout, control: $control); $this->responseStatus = $response->getStatusCode(); - return $this->decodeJsonResponse($response, $poll); + $acceptance = PollTransport::taskKind($method, $uri) !== null + && str_ends_with((string) parse_url($uri, PHP_URL_PATH), '/offers') + && is_string($body['task_offer_token'] ?? null) && $body['task_offer_token'] !== ''; + + return $this->decodeJsonResponse($response, $poll, $acceptance); } catch (TransportException $exception) { throw $exception; } catch (JsonException $exception) { @@ -169,15 +173,16 @@ private function pollTimeoutSeconds(?array $body): int|float } /** @return array|list|null */ - private function decodeJsonResponse(ResponseInterface $response, bool $poll = false): ?array + private function decodeJsonResponse(ResponseInterface $response, bool $poll = false, bool $acceptance = false): ?array { $rawBody = (string) $response->getBody(); $status = $response->getStatusCode(); $mediaType = strtolower(trim(explode(';', $response->getHeaderLine('Content-Type'), 2)[0])); + $proof = $acceptance ? TaskDeliveryRefusal::fromJson($status, $rawBody, $poll && $mediaType === PollTransport::MEDIA_TYPE) : null; if ($poll && $mediaType === PollTransport::MEDIA_TYPE) { $frame = PollTransport::decode($rawBody, $status); if ($frame['status'] >= 300) { - throw TransportException::fromResponse($frame['status'], $frame['body'], $rawBody, $frame['headers']); + throw $this->responseFailure($frame['status'], $frame['body'], $rawBody, $frame['headers'], $acceptance, $proof); } return $frame['body']; @@ -186,7 +191,7 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa // An upstream proxy can return HTML or an empty body. Preserve // its status without interpreting it as a successful protocol reply. $decoded = json_decode($rawBody, true); - throw TransportException::fromResponse($status, is_array($decoded) ? $decoded : null, $rawBody); + throw $this->responseFailure($status, is_array($decoded) ? $decoded : null, $rawBody, [], $acceptance, $proof); } $decoded = $rawBody === '' ? null : json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR); @@ -197,6 +202,17 @@ private function decodeJsonResponse(ResponseInterface $response, bool $poll = fa return $decoded; } + /** + * @param array|list|null $body + * @param array $headers + */ + private function responseFailure(int $status, ?array $body, string $raw, array $headers, bool $acceptance, ?TaskDeliveryRefusal $proof): TransportException + { + return $acceptance + ? new TransportException('Task acceptance is not confirmed.', $status, $body, headers: $headers, taskDeliveryRefusal: $proof) + : TransportException::fromResponse($status, $body, $raw, $headers); + } + /** * @param array $headers * @param array|null $body diff --git a/src/Transport/TaskDeliveryRefusal.php b/src/Transport/TaskDeliveryRefusal.php new file mode 100644 index 00000000..574a8d9f --- /dev/null +++ b/src/Transport/TaskDeliveryRefusal.php @@ -0,0 +1,45 @@ + 'task_offer_expired', + 'stale_worker_registration' => 'worker_registration_lost_authority', + ]; + + private function __construct( + public readonly int $status, + public readonly string $pollStatus, + public readonly string $reason, + ) { + } + + public static function fromJson(int $status, string $raw, bool $progress = false): ?self + { + $body = UniqueJson::decode($raw); + if ($progress) { + $frame = PollTransport::decode($raw, $status); + $status = $frame['status']; + $body = $frame['body']; + } + if ($status !== 409 || !is_array($body) || count($body) !== 3 + || !array_key_exists('task', $body) || $body['task'] !== null + || !is_string($body['poll_status'] ?? null) || !is_string($body['reason'] ?? null) + || (self::PAIRS[$body['poll_status']] ?? null) !== $body['reason']) { + return null; + } + + return new self($status, $body['poll_status'], $body['reason']); + } + + /** @return array{task: null, poll_status: string, reason: string} */ + public function body(): array + { + return ['task' => null, 'poll_status' => $this->pollStatus, 'reason' => $this->reason]; + } +} diff --git a/src/Transport/UniqueJson.php b/src/Transport/UniqueJson.php new file mode 100644 index 00000000..a15cf743 --- /dev/null +++ b/src/Transport/UniqueJson.php @@ -0,0 +1,133 @@ += 512) { + throw new JsonException('Maximum JSON depth exceeded.'); + } + self::whitespace($json, $offset); + $token = $json[$offset] ?? ''; + if ($token === '{') { + self::object($json, $offset, $depth + 1); + } elseif ($token === '[') { + self::items($json, $offset, $depth + 1); + } elseif ($token === '"') { + self::stringToken($json, $offset); + } elseif (preg_match('/\G(?:true|false|null)/A', $json, $match, 0, $offset) === 1) { + $offset += strlen($match[0]); + } elseif (preg_match('/\G-?(?:0|[1-9]\d*)(?:\.\d+)?/A', $json, $match, 0, $offset) === 1) { + $offset += strlen($match[0]); + if (preg_match('/\G[eE][+-]?\d+/A', $json, $exponent, 0, $offset) === 1) { + $offset += strlen($exponent[0]); + } + } else { + throw new JsonException('Invalid JSON value.'); + } + } + + private static function object(string $json, int &$offset, int $depth): void + { + ++$offset; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') === '}') { + ++$offset; + return; + } + $seen = []; + while (true) { + if (($json[$offset] ?? '') !== '"') { + throw new JsonException('Invalid JSON object member.'); + } + $key = json_decode(self::stringToken($json, $offset), true, 512, JSON_THROW_ON_ERROR); + $identity = "\0".$key; + if (isset($seen[$identity])) { + throw new JsonException('Duplicate JSON object member.'); + } + $seen[$identity] = true; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') !== ':') { + throw new JsonException('Invalid JSON object separator.'); + } + ++$offset; + self::value($json, $offset, $depth); + self::whitespace($json, $offset); + $separator = $json[$offset] ?? ''; + ++$offset; + if ($separator === '}') { + return; + } + if ($separator !== ',') { + throw new JsonException('Invalid JSON object separator.'); + } + self::whitespace($json, $offset); + } + } + + private static function items(string $json, int &$offset, int $depth): void + { + ++$offset; + self::whitespace($json, $offset); + if (($json[$offset] ?? '') === ']') { + ++$offset; + return; + } + while (true) { + self::value($json, $offset, $depth); + self::whitespace($json, $offset); + $separator = $json[$offset] ?? ''; + ++$offset; + if ($separator === ']') { + return; + } + if ($separator !== ',') { + throw new JsonException('Invalid JSON array separator.'); + } + } + } + + private static function stringToken(string $json, int &$offset): string + { + $start = $offset++; + while (($end = strpos($json, '"', $offset)) !== false) { + $slashes = 0; + for ($index = $end - 1; $index > $start && $json[$index] === '\\'; --$index) { + ++$slashes; + } + $offset = $end + 1; + if ($slashes % 2 === 0) { + return substr($json, $start, $offset - $start); + } + } + + throw new JsonException('Unterminated JSON string.'); + } + + private static function whitespace(string $json, int &$offset): void + { + $offset += strspn($json, " \t\r\n", $offset); + } +} diff --git a/src/Worker.php b/src/Worker.php index 33993943..f4d82243 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -667,6 +667,7 @@ private function registerWithRetry(): ?array 'worker_sessions', 'sticky_execution', 'cross_kind_poll_wake', + ...($this->client->workerProtocolVersion === '1.20' ? [\DurableWorkflow\Worker\TaskDelivery::CAPABILITY] : []), ...($this->enableCooperativeCancellation ? ['cooperative_cancellation', 'nexus_cooperative_cancellation'] : []), ...($this->enablePreparedLocalActivities ? ['prepared_local_activities'] : []), ...($this->preparedLocalActivityGroupsSupported ? ['prepared_local_activity_groups'] : []), @@ -942,6 +943,7 @@ private function pollWithRetry(string $taskKind, \Closure $poll, int $requestedT } if (!$exception->isTransientConnectionFailure() && !$exception->isTransientUpstreamFailure() + && !$exception instanceof \DurableWorkflow\Exception\TaskDeliveryUncertain && !PollResponse::isTransientPayloadFailure($exception) && !$databaseUnavailable && !$storageAdmission) { $requestId = 'php-'.$taskKind.'-poll-'.bin2hex(random_bytes(16)); diff --git a/src/Worker/PollResponse.php b/src/Worker/PollResponse.php index e09477df..62a4c261 100644 --- a/src/Worker/PollResponse.php +++ b/src/Worker/PollResponse.php @@ -6,6 +6,7 @@ use DurableWorkflow\Exception\ExternalPayloadException; use DurableWorkflow\Exception\ServerException; +use DurableWorkflow\Exception\TaskDeliveryUncertain; /** Classifies full worker poll response envelopes using protocol fields. */ final class PollResponse @@ -23,6 +24,7 @@ final class PollResponse 'worker_heartbeat_stale', 'worker_draining', 'worker_stopped', + 'worker_registration_lost_authority', ]; /** @param array $response */ @@ -42,7 +44,8 @@ public static function isTerminal(array $response): bool */ public static function isTransientFailure(ServerException $exception): bool { - if ($exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() + if ($exception instanceof TaskDeliveryUncertain + || $exception->isTransientConnectionFailure() || $exception->isTransientUpstreamFailure() || self::isTransientPayloadFailure($exception)) { return true; } diff --git a/src/Worker/TaskDelivery.php b/src/Worker/TaskDelivery.php new file mode 100644 index 00000000..606dd6e6 --- /dev/null +++ b/src/Worker/TaskDelivery.php @@ -0,0 +1,254 @@ + */ + private array $registrations = []; + /** @var array */ + private array $pending = []; + /** @var array> */ + private array $accepting = []; + + /** + * @param list $capabilities + * @param array $response + */ + public function registered(string $workerId, string $queue, string $protocol, array $capabilities, array $response): void + { + $server = $response['server_capabilities'] ?? []; + $feature = is_array($server) ? ($server['task_delivery'] ?? null) : null; + if ($protocol !== '1.20' || !in_array(self::CAPABILITY, $capabilities, true) + || $feature === null || (is_array($feature) && ($feature['supported'] ?? null) === false)) { + foreach ($this->pending as $attempt) { + if ($attempt->workerId === $workerId) { + throw new ServerException('An unresolved delivery cannot change transport modes.', 201, 'invalid_task_delivery_capability'); + } + } + $this->forget($workerId); + return; + } + $token = $response['registration_token'] ?? null; + if (!is_array($feature) || array_is_list($feature) || ($feature['supported'] ?? null) !== true + || ($feature['schema'] ?? null) !== self::SCHEMA || ($feature['version'] ?? null) !== self::VERSION + || ($feature['header'] ?? null) !== self::HEADER || ($feature['worker_capability'] ?? null) !== self::CAPABILITY + || ($feature['minimum_protocol_version'] ?? null) !== '1.20' || ($feature['endpoint_suffix'] ?? null) !== '/offers' + || ($feature['unaccepted_offer_reserves_task'] ?? null) !== false + || !is_int($feature['acceptance_timeout_seconds'] ?? null) + || $feature['acceptance_timeout_seconds'] < 1 || $feature['acceptance_timeout_seconds'] > 10 + || !is_string($token) || preg_match('/^[a-f0-9]{32}$/D', $token) !== 1) { + throw new ServerException('The task-delivery registration contract is invalid.', 201, 'invalid_task_delivery_capability'); + } + if (($this->registrations[$workerId] ?? null) !== ['queue' => $queue, 'token' => $token]) { + $this->forget($workerId); + } + $this->registrations[$workerId] = ['queue' => $queue, 'token' => $token]; + } + + /** + * @param array $body + * @param Closure(string, array, ?TaskDeliveryAttempt): array $send + * @return array|null Null means negotiation never selected this mode. + */ + public function poll(string $path, array $body, Closure $send): ?array + { + $workerId = (string) $body['worker_id']; + $registration = $this->registrations[$workerId] ?? null; + if ($registration === null || $registration['queue'] !== ($body['task_queue'] ?? null)) { + return null; + } + $allowedKinds = self::requestedKinds($path, $body); + if (array_key_exists('task_kinds', $body)) { + $body['task_kinds'] = $allowedKinds; + } + $key = hash('sha256', json_encode([$path, $workerId, $body['task_queue'], $body['poll_request_id']], JSON_THROW_ON_ERROR)); + $selected = $this->pending[$key] ?? null; + if (($selected !== null && !$selected->isForKinds($allowedKinds)) + || (isset($this->accepting[$key]) && $this->accepting[$key] !== $allowedKinds)) { + throw new InvalidArgumentException('A logical task poll cannot change its requested task kinds.'); + } + if (isset($this->accepting[$key])) { + return ['task' => null]; + } + // Hold ownership across both requests, including a suspended offer read. + $this->accepting[$key] = $allowedKinds; + try { + $body['registration_token'] = $registration['token']; + + return $this->deliver($key, substr($path, 0, -5).'/offers', $body, $allowedKinds, $selected, $send); + } finally { + unset($this->accepting[$key]); + } + } + + /** + * @param array $body + * @param list $allowedKinds + * @param Closure(string, array, ?TaskDeliveryAttempt): array $send + * @return array + */ + private function deliver(string $key, string $wirePath, array $body, array $allowedKinds, ?TaskDeliveryAttempt $selected, Closure $send): array + { + try { + if ($selected === null) { + $response = $send($wirePath, $body, null); + $this->assertCurrentRegistration($body); + if (!array_key_exists('task_offer', $response)) { + return $response; + } + $candidate = TaskDeliveryAttempt::fromResponse($wirePath, $body, $response, $allowedKinds); + $selected = $this->pending[$key] ??= $candidate; + } + $response = $send($selected->path, $selected->requestBody(), $selected); + $this->assertCurrentRegistration($selected->requestBody()); + if ($response['task'] === null) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], 200); + } + } catch (ServerException $exception) { + $response = $selected === null ? null : $this->refusedResponse($exception, $selected->requestBody()); + if ($response === null) { + if ($this->isUncertain($exception, $selected)) { + throw new TaskDeliveryUncertain((string) $body['poll_request_id'], $exception->status, $exception); + } + throw $exception; + } + } + $this->resolved($key, $selected); + + return $response; + } + + /** @internal Recheck after I/O and before observing or returning its response. + * @param array $body + */ + public function assertCurrentRegistration(array $body): void + { + $current = $this->registrations[(string) $body['worker_id']] ?? null; + if ($current === null || $current !== ['queue' => $body['task_queue'] ?? null, 'token' => $body['registration_token'] ?? null]) { + throw new ServerException('The task-delivery registration changed before its response.', 200, + 'stale_task_delivery_registration'); + } + } + + /** + * Validate before payload hydration can initialize a claim supervisor. + * @param array $response + * @param array $body + */ + public static function assertResponse(array $response, array $body, string $path, ?TaskDeliveryAttempt $selected): void + { + $task = $response['task'] ?? null; + $kind = $selected?->taskKind; + $idField = match ($kind) { + 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id', + }; + if (!array_key_exists('task', $response) + || (($selected !== null || $task !== null) && array_key_exists('task_offer', $response)) + || (($response['poll_status'] ?? null) === 'offered' && !array_key_exists('task_offer', $response)) + || ($task !== null && ($selected === null || !is_array($task) || array_is_list($task) + || !is_string($task[$idField] ?? null) || $task[$idField] === '' + || ($task['lease_owner'] ?? null) !== $body['worker_id'] + || $task[$idField] !== $selected->taskId + || ($task['task_kind'] ?? null) !== $kind + || !$selected->isForKinds(self::requestedKinds($path, $body))))) { + throw new ServerException('The task-delivery response is invalid.', 200, 'invalid_task_delivery_response'); + } + } + + /** + * @param array $body + * @return list + */ + private static function requestedKinds(string $path, array $body): array + { + $kind = str_replace('-', '_', (string) PollTransport::taskKind('POST', $path)); + if (!in_array($kind, ['workflow', 'activity', 'query', 'update_validation'], true)) { + throw new InvalidArgumentException('Task delivery requires a supported poll endpoint.'); + } + $permitted = $kind === 'workflow' ? ['workflow', 'update_validation'] : [$kind]; + $requested = array_key_exists('task_kinds', $body) ? $body['task_kinds'] : [$kind]; + if (!is_array($requested) || !array_is_list($requested) || $requested === []) { + throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); + } + $normalized = []; + foreach ($requested as $value) { + if (!is_string($value) || !in_array($value, $permitted, true)) { + throw new InvalidArgumentException('Task kinds must be a nonempty list allowed by the poll endpoint.'); + } + // Copy scalar values rather than retaining caller-owned array references. + $normalized[] = $value; + } + $normalized = array_values(array_unique($normalized)); + sort($normalized, SORT_STRING); + + return $normalized; + } + + public function forget(string $workerId): void + { + unset($this->registrations[$workerId]); + foreach ($this->pending as $key => $attempt) { + if ($attempt->workerId === $workerId) { + unset($this->pending[$key]); + } + } + } + + /** @return array */ + public function __debugInfo(): array + { + return ['registered_workers' => count($this->registrations), 'unresolved_deliveries' => count($this->pending)]; + } + + /** + * @param array $request + * @return array|null + */ + private function refusedResponse(ServerException $exception, array $request): ?array + { + if (!TaskDeliveryRefusal::matches($exception)) { + return null; + } + $response = $exception->details ?? []; + if ($exception->reason === 'task_offer_expired') { + $response['poll_status'] = 'empty'; + } else { + $workerId = (string) $request['worker_id']; + $current = $this->registrations[$workerId] ?? null; + if ($current !== null && $current['token'] === ($request['registration_token'] ?? null) + && $current['queue'] === ($request['task_queue'] ?? null)) { + $this->forget($workerId); + } + } + + return $response; + } + + private function isUncertain(ServerException $exception, ?TaskDeliveryAttempt $selected): bool + { + return !$exception instanceof TaskDeliveryUncertain && !PollResponse::isTransientPayloadFailure($exception) + && ($selected !== null || in_array($exception->status, [404, 405], true)); + } + + private function resolved(string $key, ?TaskDeliveryAttempt $selected): void + { + if (($this->pending[$key] ?? null) === $selected) { + unset($this->pending[$key]); + } + } +} diff --git a/src/Worker/TaskDeliveryAttempt.php b/src/Worker/TaskDeliveryAttempt.php new file mode 100644 index 00000000..636458b4 --- /dev/null +++ b/src/Worker/TaskDeliveryAttempt.php @@ -0,0 +1,66 @@ + $allowedKinds + * @param array $body + */ + private function __construct( + public readonly string $workerId, + public readonly string $path, + public readonly string $taskId, + public readonly string $taskKind, + private readonly array $allowedKinds, + private readonly array $body, + ) { + } + + /** + * @param array $body + * @param array $response + * @param list $allowedKinds + */ + public static function fromResponse(string $path, array $body, array $response, array $allowedKinds): self + { + $offer = $response['task_offer'] ?? null; + if (!array_key_exists('task', $response) || $response['task'] !== null || ($response['poll_status'] ?? null) !== 'offered' + || !is_array($offer) || array_is_list($offer) + || ($offer['schema'] ?? null) !== TaskDelivery::SCHEMA + || !is_string($offer['task_kind'] ?? null) + || !in_array($offer['task_kind'], $allowedKinds, true) + || !is_string($offer['task_id'] ?? null) || $offer['task_id'] === '' + || !is_string($offer['token'] ?? null) || $offer['token'] === '' || strlen($offer['token']) > 8192 + || !is_int($offer['expires_at_ms'] ?? null) || $offer['expires_at_ms'] < 1) { + throw new ServerException('The server returned an invalid task offer.', 200, 'invalid_task_delivery_offer'); + } + + return new self((string) $body['worker_id'], $path, $offer['task_id'], $offer['task_kind'], $allowedKinds, + array_replace($body, ['task_offer_token' => $offer['token'], 'timeout_seconds' => 0])); + } + + /** @param list $allowedKinds */ + public function isForKinds(array $allowedKinds): bool + { + return $this->allowedKinds === $allowedKinds; + } + + /** @return array */ + public function requestBody(): array + { + return $this->body; + } + + /** @return array */ + public function __debugInfo(): array + { + return ['worker_id' => $this->workerId, 'path' => $this->path, 'task_id' => $this->taskId]; + } +} diff --git a/src/Worker/TaskDeliveryRefusal.php b/src/Worker/TaskDeliveryRefusal.php new file mode 100644 index 00000000..2cb5b295 --- /dev/null +++ b/src/Worker/TaskDeliveryRefusal.php @@ -0,0 +1,28 @@ +getPrevious(); + if (!$transport instanceof TransportException || $transport->taskDeliveryRefusal === null) { + return false; + } + $proof = $transport->taskDeliveryRefusal; + $response = $exception->details; + + return $transport->status === $proof->status && $exception->status === $proof->status + && $exception->reason === $proof->reason && $response !== null && count($response) === 3 + && array_key_exists('task', $response) && $response['task'] === null + && ($response['poll_status'] ?? null) === $proof->pollStatus + && ($response['reason'] ?? null) === $proof->reason; + } +} diff --git a/tests/ClientContractTest.php b/tests/ClientContractTest.php index 439a9c90..eef3ec12 100644 --- a/tests/ClientContractTest.php +++ b/tests/ClientContractTest.php @@ -119,6 +119,7 @@ public function testDefaultWorkerRegistrationAdvertisesTheCompletePhpCapabilityS 'memo_upserts', 'typed_search_attributes', 'durable_selection', + 'task_delivery_offers', ], 'capability_manifest' => [ 'local_activities' => [ @@ -140,6 +141,7 @@ public function testDefaultWorkerRegistrationAdvertisesTheCompletePhpCapabilityS 'max_concurrent_workflow_tasks' => 1, 'max_concurrent_activity_tasks' => 1, 'max_concurrent_worker_sessions' => 10, + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ], ], $transport->requests[0]); } diff --git a/tests/UniqueJsonNumberTest.php b/tests/UniqueJsonNumberTest.php new file mode 100644 index 00000000..ffbfd315 --- /dev/null +++ b/tests/UniqueJsonNumberTest.php @@ -0,0 +1,36 @@ +expectException(JsonException::class); + UniqueJson::decode($json); + } + + /** @return iterable */ + public static function invalidNumbers(): iterable + { + foreach (['١', '１', '1١', '1.١', '1e١', '-١', '1e+１', '[1,١]', '{"n":1.१}', + '01', '1.', '1e', '1e+', 'NaN', 'Infinity'] as $json) { + yield $json => [$json]; + } + } +} diff --git a/tests/WorkerAuthoringTest.php b/tests/WorkerAuthoringTest.php index b8918bfe..930c2af6 100644 --- a/tests/WorkerAuthoringTest.php +++ b/tests/WorkerAuthoringTest.php @@ -182,6 +182,7 @@ public function testHighLevelWorkerAdvertisesTheCompleteManagedCapabilitySet(): 'worker_sessions', 'sticky_execution', 'cross_kind_poll_wake', + 'task_delivery_offers', ], 'capability_manifest' => [ 'local_activities' => [ @@ -204,6 +205,7 @@ public function testHighLevelWorkerAdvertisesTheCompleteManagedCapabilitySet(): 'max_concurrent_activity_tasks' => 1, 'max_concurrent_worker_sessions' => 10, 'build_id' => 'release-a', + 'process_metrics' => ['process_id' => getmypid(), 'process_uptime_seconds' => 0], ], ], $transport->requests[0]); } diff --git a/tests/WorkerPollTransportTest.php b/tests/WorkerPollTransportTest.php index eda12b19..2ecf2df0 100644 --- a/tests/WorkerPollTransportTest.php +++ b/tests/WorkerPollTransportTest.php @@ -51,6 +51,27 @@ public static function pollKinds(): array return [['workflow'], ['activity'], ['query'], ['update-validation']]; } + #[DataProvider('pollKinds')] + public function testOfferAndAcceptanceKeepProgressNegotiationAndTheirAbsoluteBudgets(string $kind): void + { + $calls = []; + $http = new GuzzleClient(['handler' => static function (RequestInterface $request, array $options) use (&$calls) { + $calls[] = [$request, $options]; + return Create::promiseFor(new Response(200, ['Content-Type' => self::MEDIA_TYPE], self::frame(200))); + }]); + $transport = new Psr18Transport($http); + $uri = 'https://server.example/api/worker/'.$kind.'-tasks/offers'; + $transport->send('POST', $uri, [], ['timeout_seconds' => 4, 'poll_request_id' => 'same-poll']); + $transport->send('POST', $uri, [], ['timeout_seconds' => 0, 'poll_request_id' => 'same-poll', 'task_offer_token' => 'same-packet']); + self::assertSame(9, $calls[0][1]['timeout']); + self::assertSame(5, $calls[1][1]['timeout']); + foreach ($calls as [$request, $options]) { + self::assertSame(self::MEDIA_TYPE.', application/json;q=0.9', $request->getHeaderLine('Accept')); + self::assertFalse($options['stream']); + } + self::assertSame('same-packet', json_decode((string) $calls[1][0]->getBody(), true)['task_offer_token']); + } + public function testAShorterInjectedGuzzleDeadlineIsPreserved(): void { $observed = null; diff --git a/tests/WorkerTaskDeliveryKindsTest.php b/tests/WorkerTaskDeliveryKindsTest.php new file mode 100644 index 00000000..fe073413 --- /dev/null +++ b/tests/WorkerTaskDeliveryKindsTest.php @@ -0,0 +1,350 @@ + str_starts_with($id, 'kind-')))); + } + + /** @param list|null $kinds */ + #[DataProvider('validKinds')] + public function testOnlyRequestedKindsCanProduceTheirSelectedReceipt(?array $kinds, string $selected, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), self::offer($selected), self::claimed($selected)]); + $client = self::client($transport); + [$result, $error] = self::capture(static fn (): array => self::poll($client, $kinds, $endpoint)); + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame(self::claimed($selected), $result); + self::assertCount(3, $transport->requests); + $offer = $transport->requests[1]; + $accept = $transport->requests[2]; + self::assertStringEndsWith('/'.$endpoint.'-tasks/offers', $offer['uri']); + self::assertSame($offer['uri'], $accept['uri']); + $originalBody = $offer['body']; + self::assertIsArray($originalBody); + self::assertSame(array_replace($originalBody, ['timeout_seconds' => 0, 'task_offer_token' => 'selected-packet']), $accept['body']); + if ($kinds !== null) { + self::assertSame(in_array('workflow', $kinds, true) && in_array('update_validation', $kinds, true) ? self::BOTH : $kinds, + $offer['body']['task_kinds']); + } + } + + /** @return iterable|null, string, string}> */ + public static function validKinds(): iterable + { + $data = json_decode((string) file_get_contents(self::FIXTURE), true, flags: JSON_THROW_ON_ERROR); + foreach ($data['cases'] as $case) { + if (in_array($case['id'], ['kind-workflow', 'kind-validator', 'kind-mixed-order'], true)) { + foreach ($case['request_kind_orders'] ?? [$case['requested_kinds']] as $index => $kinds) { + yield $case['id'].' / '.$index => [$kinds, $case['offered_kind'], $case['endpoint']]; + } + } + } + yield 'mixed workflow / order 1' => [self::BOTH, 'workflow', 'workflow']; + yield 'mixed workflow / order 2' => [['workflow', 'update_validation'], 'workflow', 'workflow']; + yield 'normalized duplicate kinds' => [['workflow', 'update_validation', 'workflow'], 'update_validation', 'workflow']; + yield 'default workflow set' => [null, 'workflow', 'workflow']; + foreach (['activity', 'query', 'update-validation'] as $endpoint) { + $kind = str_replace('-', '_', $endpoint); + yield 'single-kind '.$endpoint => [[$kind], $kind, $endpoint]; + } + } + + /** @param list $kinds */ + #[DataProvider('unrequestedKinds')] + public function testAnUnrequestedOfferSendsNoAcceptance(array $kinds, string $offered, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), self::offer($offered), self::claimed($offered)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, $kinds, $endpoint)); + self::assertInstanceOf(ServerException::class, $error); + self::assertSame('invalid_task_delivery_offer', $error->reason); + self::assertCount(2, $transport->requests); + } + + /** @return iterable, string, string}> */ + public static function unrequestedKinds(): iterable + { + yield 'kind-unrequested' => [['workflow'], 'update_validation', 'workflow']; + yield 'validator-only cannot select workflow' => [['update_validation'], 'workflow', 'workflow']; + foreach (['activity', 'query', 'update-validation'] as $endpoint) { + yield $endpoint => [[str_replace('-', '_', $endpoint)], 'workflow', $endpoint]; + } + } + + #[DataProvider('receiptMutations')] + public function testSubstitutedOrMissingReceiptIdentityCannotReleaseThePacket(string $kind, string $mutation): void + { + $bad = self::claimed($kind); + $field = $kind === 'update_validation' ? 'update_validation_task_id' : 'task_id'; + if ($mutation === 'kind-substitution') { + $bad['task']['task_kind'] = $kind === 'workflow' ? 'update_validation' : 'workflow'; + $bad['task']['task_id'] = $bad['task']['update_validation_task_id'] = 'selected-task'; + } elseif ($mutation === 'missing-kind') { + unset($bad['task']['task_kind']); + } elseif ($mutation === 'null-kind') { + $bad['task']['task_kind'] = null; + } elseif ($mutation === 'task-substitution') { + $bad['task'][$field] = 'other-task'; + } else { + unset($bad['task'][$field]); + $bad['task'][$field === 'task_id' ? 'update_validation_task_id' : 'task_id'] = 'selected-task'; + } + $transport = new FakeTransport([self::registration(), self::offer($kind), $bad, self::claimed($kind)]); + $observed = 0; + $client = self::client($transport)->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(0, $observed); + self::assertSame(self::claimed($kind), self::poll(clone $client, ['workflow', 'update_validation'])); + self::assertSame(1, $observed); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + /** @return iterable */ + public static function receiptMutations(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (['kind-substitution', 'missing-kind', 'null-kind', 'task-substitution', 'wrong-id-field'] as $mutation) { + yield $kind.' / '.$mutation => [$kind, $mutation]; + } + } + } + + #[DataProvider('selectedKinds')] + public function testChangedKindsCannotReplaceAnUncertainOriginalRequest(string $kind): void + { + $transport = new FakeTransport([self::registration(), self::offer($kind), new TransportException('Timeout.', 504), self::claimed($kind)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + $clone = clone $client; + [, $error] = self::capture(static fn (): array => self::poll($clone, ['workflow'])); + self::assertInstanceOf(InvalidArgumentException::class, $error); + self::assertCount(3, $transport->requests); + self::assertSame(self::claimed($kind), self::poll($clone, ['workflow', 'update_validation', 'workflow'])); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + /** @return array */ + public static function selectedKinds(): array { return [['workflow'], ['update_validation']]; } + + public function testCallerReferencesCannotMutateTheSelectedKindSetOrRetryBytes(): void + { + $kind = 'workflow'; + $requested = [&$kind, 'update_validation']; + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Timeout.', 504), self::claimed('workflow')]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, $requested)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + $originalPacket = json_encode($transport->requests[2], JSON_THROW_ON_ERROR); + $kind = 'activity'; + [$result, $error] = self::capture(static fn (): array => self::poll(clone $client, self::BOTH)); + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame(self::claimed('workflow'), $result); + self::assertSame($originalPacket, json_encode($transport->requests[3], JSON_THROW_ON_ERROR)); + } + + #[DataProvider('concurrentKinds')] + public function testKindSetBindingSpansOfferAndAcceptanceOwnership(string $kind, string $phase, bool $clone, bool $changed): void + { + $offers = $accepts = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ( + $kind, $phase, &$offers, &$accepts, + ): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (isset($body['task_offer_token'])) { + ++$accepts; + if ($phase === 'accept') { \Fiber::suspend('accept'); } + return self::claimed($kind); + } + ++$offers; + if ($phase === 'offer') { \Fiber::suspend('offer'); } + return self::offer($kind); + }); + $client = self::client($transport); + $owner = new \Fiber(static fn (): array => self::poll($client, self::BOTH)); + try { $paused = $owner->start(); } + catch (ServerException $error) { self::fail('The valid owner was refused before its pause: '.$error->reason); } + self::assertSame($phase, $paused); + $other = $clone ? clone $client : $client; + [$result, $error] = self::capture(static fn (): array => self::poll($other, $changed ? ['workflow'] : ['workflow', 'update_validation'])); + try { $owner->resume(); } + catch (ServerException $error) { self::fail('The valid owner was refused after its pause: '.$error->reason); } + self::assertSame(self::claimed($kind), $owner->getReturn()); + if ($changed) { + self::assertInstanceOf(InvalidArgumentException::class, $error); + } else { + self::assertNull($error); + self::assertSame(['task' => null], $result); + } + self::assertSame(1, $offers); + self::assertSame(1, $accepts); + self::assertCount(3, $transport->requests); + } + + /** @return iterable */ + public static function concurrentKinds(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (['offer', 'accept'] as $phase) { + foreach ([false, true] as $clone) { + foreach ([false, true] as $changed) { + yield $kind.' / '.$phase.' / '.(int) $clone.' / '.(int) $changed => [$kind, $phase, $clone, $changed]; + } + } + } + } + } + + #[DataProvider('invalidKindSets')] + public function testMalformedKindSetsRefuseBeforeAnOffer(mixed $kinds, string $endpoint): void + { + $transport = new FakeTransport([self::registration(), ['task' => null]]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::rawPoll($client, ['task_kinds' => $kinds], $endpoint)); + self::assertInstanceOf(InvalidArgumentException::class, $error); + self::assertCount(1, $transport->requests); + } + + /** @return iterable */ + public static function invalidKindSets(): iterable + { + foreach ([null, '', [], ['kind' => 'workflow'], [1], ['activity'], ['update-validation'], ['workflow', 'foreign']] as $index => $kinds) { + yield (string) $index => [$kinds, 'workflow']; + } + foreach (['activity', 'query', 'update-validation', 'unknown'] as $endpoint) { + yield $endpoint => [['workflow'], $endpoint]; + } + } + + /** @param TransportException|array|null $outcome */ + #[DataProvider('uncertainKinds')] + public function testUncertainMixedRepliesRetainTheExactOriginalPacket(string $kind, TransportException|array|null $outcome): void + { + $transport = new FakeTransport([self::registration(), self::offer($kind), $outcome, self::claimed($kind)]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, self::BOTH)); + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(self::claimed($kind), self::poll(clone $client, ['workflow', 'update_validation'])); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertStringEndsWith('/workflow-tasks/offers', $transport->requests[3]['uri']); + } + + /** @return iterable|null}> */ + public static function uncertainKinds(): iterable + { + foreach (['workflow', 'update_validation'] as $kind) { + foreach (WorkerTaskDeliveryTest::uncertainOutcomes() as $name => [$outcome]) { + yield $kind.' / '.$name => [$kind, $outcome]; + } + } + } + + public function testAnExecutableTaskWithoutASelectedOfferCannotBypassKindBinding(): void + { + $transport = new FakeTransport([self::registration(), self::claimed('workflow')]); + $client = self::client($transport); + [, $error] = self::capture(static fn (): array => self::poll($client, ['workflow'])); + self::assertInstanceOf(ServerException::class, $error); + self::assertSame('invalid_task_delivery_response', $error->reason); + self::assertCount(2, $transport->requests); + } + + /** + * @param list|null $kinds + * @return array + */ + private static function poll(Client $client, ?array $kinds, string $endpoint = 'workflow'): array + { + return self::rawPoll($client, $kinds === null ? [] : ['task_kinds' => $kinds], $endpoint); + } + + /** + * @param array $extra + * @return array + */ + private static function rawPoll(Client $client, array $extra, string $endpoint = 'workflow'): array + { + // Exercise the same logical-request boundary used by the public poll + // methods, without adding update-validator authoring to this source slice. + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => 'orders', 'poll_request_id' => 'kind-poll', 'timeout_seconds' => 4, + ...$extra, + ]); + } + + /** @return array{array|null, ServerException|InvalidArgumentException|null} */ + private static function capture(Closure $operation): array + { + try { return [$operation(), null]; } + catch (ServerException|InvalidArgumentException $error) { return [null, $error]; } + } + + private static function client(FakeTransport $transport): Client + { + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + return $client; + } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, + 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'selected-task', + 'token' => 'selected-packet', 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => 'selected-task', 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } +} diff --git a/tests/WorkerTaskDeliveryRefusalTest.php b/tests/WorkerTaskDeliveryRefusalTest.php new file mode 100644 index 00000000..2ae3fb0e --- /dev/null +++ b/tests/WorkerTaskDeliveryRefusalTest.php @@ -0,0 +1,397 @@ + $response */ + #[DataProvider('malformedRefusals')] + public function testMalformed409RetainsThePacketAndNegotiatedRegistration(array $response): void + { + self::assertUncertainReply(self::wireFailure($response)); + } + + /** @return iterable}> */ + public static function malformedRefusals(): iterable + { + $pairs = self::existingPairs(); + $statuses = array_values(array_unique([...array_column($pairs, 0), 'draining', 'stopped', + 'worker_heartbeat_stale', 'worker_registration_lost_authority', 'empty', 'offered', 'foreign_status'])); + $reasons = array_values(array_unique([...array_column($pairs, 1), 'worker_draining', 'worker_stopped', + 'stale_worker_registration', 'worker_heartbeat_stale', 'backend_unavailable', 'unrelated_reason'])); + foreach ($statuses as $status) { + foreach ($reasons as $reason) { + if (!in_array([$status, $reason], $pairs, true)) { + yield $status.' / '.$reason => [self::body($status, $reason)]; + } + } + } + foreach ($pairs as [$status, $reason]) { + foreach (self::contradictoryFields() as $name => [$key, $value]) { + yield $status.' / '.$name => [[...self::body($status, $reason), $key => $value]]; + } + foreach (['task', 'poll_status', 'reason'] as $key) { + $body = self::body($status, $reason); + unset($body[$key]); + yield $status.' / missing '.$key => [$body]; + } + } + } + + /** @return array */ + private static function contradictoryFields(): array + { + return [ + 'non-null task' => ['task', ['task_id' => 'unexpected']], + 'false task' => ['task', false], + 'offer null is still present' => ['task_offer', null], + 'offer object' => ['task_offer', ['task_id' => 'other-task']], + 'acceptance token' => ['task_offer_token', 'contradictory-packet'], + 'accepted marker' => ['accepted', true], + 'lease owner' => ['lease_owner', 'worker'], + 'attempt marker' => ['attempt', 1], + 'task ID outside task' => ['task_id', 'selected-task'], + 'kind outside task' => ['task_kind', 'workflow'], + 'claim retry marker' => ['retry_same_poll_request_id', true], + 'retryable wrong type' => ['retryable', 'true'], + 'retryable true' => ['retryable', true], + 'retryable false' => ['retryable', false], + 'retry delay' => ['retry_after_seconds', 1], + 'matching worker' => ['worker_id', 'worker'], + 'matching queue' => ['task_queue', 'orders'], + 'plain error text' => ['error', 'Synthetic refusal.'], + 'plain message text' => ['message', 'Synthetic refusal.'], + 'null build' => ['registered_build_id', null], + 'unknown extension' => ['unknown_terminal_extension', true], + 'foreign worker' => ['worker_id', 'other-worker'], + 'foreign queue' => ['task_queue', 'other-queue'], + 'worker wrong type' => ['worker_id', 1], + 'queue wrong type' => ['task_queue', null], + 'error wrong type' => ['error', []], + 'message wrong type' => ['message', []], + 'build wrong type' => ['registered_build_id', []], + 'status wrong type' => ['worker_status', []], + 'drain intent wrong type' => ['drain_intent', []], + 'null poll status' => ['poll_status', null], + 'null reason' => ['reason', null], + ]; + } + + /** @param array $response */ + #[DataProvider('unqualifiedSuccessReplies')] + public function testAnUnqualified200CannotResolveASelectedAcceptance(array $response): void + { + self::assertUncertainReply($response); + } + + /** @return iterable}> */ + public static function unqualifiedSuccessReplies(): iterable + { + yield 'mixed successful terminal' => [self::body('draining', 'unrelated_reason')]; + yield 'successful expiry is not the required 409' => [self::body('task_offer_expired', 'task_offer_expired')]; + yield 'successful authority loss is not the required 409' => [self::body('stale_worker_registration', 'worker_registration_lost_authority')]; + yield 'successful stopped hint' => [self::body('stopped', 'worker_stopped')]; + yield 'successful draining hint' => [self::body('draining', 'worker_draining')]; + yield 'bare legacy stale is not definitive acceptance' => [['task' => null, 'poll_status' => 'stale_worker_registration']]; + } + + #[DataProvider('wrongHttpStatuses')] + public function testAnotherHttpStatusCannotResolveTheSelectedPacket(int $status, string $pollStatus, string $reason): void + { + $body = self::body($pollStatus, $reason); + self::assertUncertainReply($status === 200 ? $body : new TransportException('Synthetic refusal.', $status, $body)); + } + + /** @return iterable */ + public static function wrongHttpStatuses(): iterable + { + foreach (self::existingPairs() as [$status, $reason]) { + foreach ([0, 200, 202, 400, 401, 403, 404, 405, 410, 429, 500, 503] as $http) { + yield $status.' / '.$http => [$http, $status, $reason]; + } + } + } + + public function testExceptionAndBodyReasonsMustAgree(): void + { + self::assertUncertainReply(new ServerException('Synthetic refusal.', 409, 'unrelated_reason', + self::body('task_offer_expired', 'task_offer_expired'), self::wireFailure(self::body('task_offer_expired', 'task_offer_expired')))); + } + + #[DataProvider('positiveRefusals')] + public function testTheExactDefinitiveRefusalPermitsOnlyItsDefinedTransition(string $status, string $reason): void + { + $body = self::body($status, $reason); + $transport = new FakeTransport([self::registration(), self::offer(), self::wireFailure($body), + self::registration(), self::offer('new-packet'), self::claimed()]); + $client = self::client($transport); + $expected = $body; + if ($reason === 'task_offer_expired') { $expected['poll_status'] = 'empty'; } + self::assertSame($expected, self::poll($client)); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed(), self::poll($client, 'new-poll')); + self::assertCount(6, $transport->requests); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[4]['body']); + self::assertSame('new-poll', $transport->requests[4]['body']['poll_request_id']); + self::assertSame('new-packet', $transport->requests[5]['body']['task_offer_token']); + } + + /** @return list */ + public static function positiveRefusals(): array { return self::existingPairs(); } + + public function testExpiryPreservesRegistrationAndAnotherUnresolvedPacket(): void + { + [$client, $transport] = self::pendingClient('poll-a', self::wireFailure(self::body('task_offer_expired', 'task_offer_expired'))); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b'); + self::assertSame('empty', self::poll($client, 'poll-a')['poll_status']); + self::assertSame(self::claimed('poll-b'), self::poll(clone $client, 'poll-b')); + self::assertSame(self::claimed('poll-c'), self::poll($client, 'poll-c')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-c', false)); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testAnUncertainLifecycleReplyCannotEraseAnotherPendingPacket(): void + { + [$client, $transport] = self::pendingClient('poll-a', new TransportException('Draining.', 409, + self::body('draining', 'worker_draining'))); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b'); + self::expectPending($client, 'poll-a'); + self::assertSame(self::claimed('poll-b'), self::poll(clone $client, 'poll-b')); + self::assertSame(self::claimed('poll-a'), self::poll($client, 'poll-a')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertIdenticalRetries($transport, 'poll-a', 3); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testDiscoveryDrainingDoesNotEraseAnAlreadySelectedPacket(): void + { + $stop = self::body('draining', 'worker_draining'); + [$client, $transport] = self::pendingClient('discovery', new TransportException('Draining.', 409, $stop), true); + self::expectPending($client, 'poll-a'); + self::assertSame($stop, self::poll($client, 'discovery')); + self::assertSame(self::claimed('poll-a'), self::poll(clone $client, 'poll-a')); + self::assertCount(1, self::packetsFor($transport, 'poll-a', false)); + self::assertIdenticalRetries($transport, 'poll-a', 2); + } + + public function testProvedAuthorityLossDoesNotForgetAnotherWorker(): void + { + $lost = self::body('stale_worker_registration', 'worker_registration_lost_authority'); + [$client, $transport] = self::pendingClient('poll-a', self::wireFailure($lost)); + $client->registerWorker('worker-b', 'orders', [], []); + self::expectPending($client, 'poll-a'); + self::expectPending($client, 'poll-b', 'worker-b'); + self::assertSame($lost, self::poll($client, 'poll-a')); + self::assertSame(self::claimed('poll-b', 'worker-b'), self::poll(clone $client, 'poll-b', 'worker-b')); + self::assertCount(1, self::packetsFor($transport, 'poll-b', false)); + self::assertIdenticalRetries($transport, 'poll-b', 2); + } + + public function testTheLegacyBareStalePollContractIsPreserved(): void + { + $response = ['task' => null, 'poll_status' => 'stale_worker_registration']; + $transport = new FakeTransport([['registered' => true], $response]); + $client = self::client($transport); + self::assertSame($response, self::poll($client)); + self::assertTrue(PollResponse::isTerminal($response)); + self::assertStringEndsWith('/workflow-tasks/poll', $transport->requests[1]['uri']); + } + + /** @param TransportException|ServerException|array $reply */ + private static function assertUncertainReply(TransportException|ServerException|array $reply): void + { + $steps = [self::registration(), self::offer(), $reply, self::claimed()]; + $transport = new FakeTransport(handler: static function () use (&$steps): ?array { + $next = array_shift($steps); + if ($next instanceof TransportException || $next instanceof ServerException) { throw $next; } + return $next; + }); + $client = self::client($transport); + $caught = null; + try { self::poll($client); } + catch (ServerException $error) { $caught = $error; } + self::assertInstanceOf(TaskDeliveryUncertain::class, $caught); + self::assertTrue(PollResponse::isTransientFailure($caught), 'An uncertain reply cannot terminate the managed worker.'); + self::assertSame(self::claimed(), self::poll(clone $client)); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertStringEndsWith('/workflow-tasks/offers', $transport->requests[3]['uri']); + } + + /** @param array $body */ + private static function wireFailure(array $body): TransportException + { + return new TransportException('Synthetic refusal.', 409, $body, + taskDeliveryRefusal: \DurableWorkflow\Transport\TaskDeliveryRefusal::fromJson(409, json_encode($body, JSON_THROW_ON_ERROR))); + } + + public function testADecodedOnlyRefusalCannotResolveThePacket(): void + { + self::assertUncertainReply(new TransportException('No original-byte evidence.', 409, + self::body('task_offer_expired', 'task_offer_expired'))); + } + + public function testAnOldAuthorityRefusalCannotEraseAReplacementIncarnation(): void + { + $registrations = 0; + $attempts = []; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use (&$registrations, &$attempts): array { + if (str_ends_with($uri, '/worker/register')) { + $response = self::registration(); + $response['registration_token'] = str_repeat(++$registrations === 1 ? 'a' : 'b', 32); + return $response; + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { return self::offer('packet-'.$poll, $poll); } + if ($poll === 'old') { + \Fiber::suspend(); + throw self::wireFailure(self::body('stale_worker_registration', 'worker_registration_lost_authority')); + } + if (!isset($attempts[$poll])) { + $attempts[$poll] = true; + throw new TransportException('Lost response.', 504); + } + return self::claimed($poll); + }); + $client = self::client($transport); + $old = new \Fiber(static fn (): array => self::poll($client, 'old')); + $old->start(); + self::assertTrue($old->isSuspended()); + $replacement = clone $client; + $replacement->registerWorker('worker', 'orders', [], []); + self::expectPending($replacement, 'new'); + $old->resume(); + self::assertSame('worker_registration_lost_authority', $old->getReturn()['reason']); + self::assertSame(self::claimed('new'), self::poll($replacement, 'new')); + self::assertCount(1, self::packetsFor($transport, 'new', false)); + self::assertIdenticalRetries($transport, 'new', 2); + self::assertSame(str_repeat('b', 32), self::packetsFor($transport, 'new', true)[1]['body']['registration_token']); + } + + /** @return list */ + private static function existingPairs(): array + { + // Exact two rows in the reviewed Server 31b281ac companion. + return [ + ['task_offer_expired', 'task_offer_expired'], + ['stale_worker_registration', 'worker_registration_lost_authority'], + ]; + } + + /** @return array */ + private static function body(string $status, string $reason): array + { + return ['task' => null, 'poll_status' => $status, 'reason' => $reason]; + } + + private static function client(FakeTransport $transport): Client + { + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + return $client; + } + + /** @return array */ + private static function poll(Client $client, string $pollId = 'refusal-poll', string $workerId = 'worker'): array + { + return $client->pollWorkflowTaskResponse($workerId, 'orders', 4, $pollId); + } + + private static function expectPending(Client $client, string $pollId, string $workerId = 'worker'): void + { + $caught = null; + try { self::poll($client, $pollId, $workerId); } + catch (ServerException $error) { $caught = $error; } + self::assertInstanceOf(TaskDeliveryUncertain::class, $caught); + } + + /** @return array{Client, FakeTransport} */ + private static function pendingClient(string $refusalPoll, TransportException $refusal, bool $discovery = false): array + { + $attempts = []; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ( + $refusalPoll, $refusal, $discovery, &$attempts, + ): array { + if (str_ends_with($uri, '/worker/register')) { + $response = self::registration(); + $response['registration_token'] = str_repeat($body['worker_id'] === 'worker' ? 'a' : 'b', 32); + return $response; + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { + if ($discovery && $poll === $refusalPoll) { throw $refusal; } + return self::offer('packet-'.$poll, $poll); + } + $attempts[$poll] = ($attempts[$poll] ?? 0) + 1; + if (in_array($poll, ['poll-a', 'poll-b'], true) && $attempts[$poll] === 1) { + throw new TransportException('Lost response.', 504); + } + if (!$discovery && $poll === $refusalPoll && $attempts[$poll] === 2) { throw $refusal; } + return self::claimed($poll, $body['worker_id']); + }); + + return [self::client($transport), $transport]; + } + + /** @return list> */ + private static function packetsFor(FakeTransport $transport, string $pollId, bool $accept): array + { + return array_values(array_filter($transport->requests, static fn (array $request): bool => + ($request['body']['poll_request_id'] ?? null) === $pollId + && isset($request['body']['task_offer_token']) === $accept)); + } + + private static function assertIdenticalRetries(FakeTransport $transport, string $pollId, int $count): void + { + $requests = self::packetsFor($transport, $pollId, true); + self::assertCount($count, $requests); + $first = json_encode($requests[0], JSON_THROW_ON_ERROR); + foreach ($requests as $request) { + self::assertSame($first, json_encode($request, JSON_THROW_ON_ERROR)); + } + } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, + 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $token = 'selected-packet', string $taskId = 'selected-task'): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => 'workflow', 'task_id' => $taskId, + 'token' => $token, 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $taskId = 'selected-task', string $workerId = 'worker'): array + { + return ['task' => ['task_id' => $taskId, 'lease_owner' => $workerId, 'task_kind' => 'workflow']]; + } +} diff --git a/tests/WorkerTaskDeliveryRegistrationTest.php b/tests/WorkerTaskDeliveryRegistrationTest.php new file mode 100644 index 00000000..5e0acb88 --- /dev/null +++ b/tests/WorkerTaskDeliveryRegistrationTest.php @@ -0,0 +1,150 @@ + 1 && in_array($change, ['token', 'unrelated'], true) ? 'b' : 'a'); + } + $poll = $body['poll_request_id']; + if (!isset($body['task_offer_token'])) { + if ($poll === 'original' && $phase === 'offer') { Fiber::suspend(); } + return self::offer($kind, $poll); + } + if ($poll === 'original' && $phase === 'accept') { Fiber::suspend(); } + if ($poll === 'replacement' && ++$replacementAttempts === 1) { + throw new TransportException('Synthetic lost response.', 504); + } + return self::claimed($kind, $poll); + }); + $observed = 0; + $client = (new Client('https://server.example', transport: $transport, + metrics: $metrics ? new RecordingMetricsRecorder() : null))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed, $phase): ?Closure { + if (($body['poll_request_id'] ?? null) === 'original' && ($response['task'] ?? null) !== null) { + ++$observed; + if ($phase === 'observer') { Fiber::suspend(); } + } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + $old = new Fiber(static function () use ($client, $endpoint, $kind): array { + try { return ['result' => self::poll($client, $endpoint, $kind, 'original')]; } + catch (ServerException $error) { return ['error' => $error]; } + }); + $old->start(); + self::assertTrue($old->isSuspended()); + $replacement = clone $client; + $queue = $change === 'queue' ? 'other-orders' : 'orders'; + $replacement->registerWorker($change === 'unrelated' ? 'other-worker' : 'worker', $queue, [], []); + try { + self::poll($replacement, $endpoint, $kind, 'replacement', $queue); + self::fail('The replacement fixture must retain its uncertain packet.'); + } catch (TaskDeliveryUncertain) { + } + $old->resume(); + $outcome = $old->getReturn(); + if (in_array($change, ['same', 'unrelated'], true)) { + self::assertSame(self::claimed($kind, 'original'), $outcome['result'] ?? null); + self::assertSame(1, $observed); + } else { + self::assertArrayNotHasKey('result', $outcome, 'A stale successful receipt cannot authorize execution.'); + self::assertInstanceOf(ServerException::class, $outcome['error'] ?? null); + if ($phase === 'observer') { + self::assertSame(1, $observed, 'The callback had already observed the payload before registration changed.'); + } else { + self::assertSame(0, $observed, 'Transport replacement must refuse before payload observation.'); + } + } + $expected = self::claimed($kind, 'replacement'); + $actual = self::poll($replacement, $endpoint, $kind, 'replacement', $queue); + self::assertSame($expected, $actual); + $packets = array_values(array_filter($transport->requests, static fn (array $request): bool => + ($request['body']['poll_request_id'] ?? null) === 'replacement' && isset($request['body']['task_offer_token']))); + self::assertCount(2, $packets); + self::assertSame($packets[0], $packets[1], 'The old response must not mutate or erase the replacement packet.'); + self::assertSame(str_repeat($change === 'token' ? 'b' : 'a', 32), $packets[1]['body']['registration_token']); + self::assertSame($queue, $packets[1]['body']['task_queue']); + } + + /** @return iterable */ + public static function registrationRaces(): iterable + { + $cases = [ + ['workflow', 'workflow', 'accept', 'token'], + ['workflow', 'update_validation', 'accept', 'token'], + ['activity', 'activity', 'accept', 'token'], + ['query', 'query', 'accept', 'token'], + ['update-validation', 'update_validation', 'accept', 'token'], + ['workflow', 'workflow', 'offer', 'token'], + ['workflow', 'workflow', 'accept', 'queue'], + ['workflow', 'workflow', 'offer', 'queue'], + ['workflow', 'workflow', 'observer', 'token'], + ['workflow', 'workflow', 'accept', 'same'], + ['workflow', 'workflow', 'accept', 'unrelated'], + ]; + foreach ($cases as $case) { + foreach ([false, true] as $metrics) { + yield implode(' / ', $case).($metrics ? ' / metrics' : '') => [...$case, $metrics]; + } + } + } + + /** @return array */ + private static function poll(Client $client, string $endpoint, string $kind, string $poll, string $queue = 'orders'): array + { + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => $queue, 'timeout_seconds' => 4, 'poll_request_id' => $poll, 'task_kinds' => [$kind], + ]); + } + + /** @return array */ + private static function registration(string $token): array + { + return ['registered' => true, 'registration_token' => str_repeat($token, 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, 'version' => TaskDelivery::VERSION, + 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind, string $poll): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => $poll, 'token' => 'synthetic-'.$poll, 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind, string $poll): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => $poll, 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } +} diff --git a/tests/WorkerTaskDeliveryTest.php b/tests/WorkerTaskDeliveryTest.php new file mode 100644 index 00000000..a7dd04c6 --- /dev/null +++ b/tests/WorkerTaskDeliveryTest.php @@ -0,0 +1,422 @@ +withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?\Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + if ($invalidFirst) { + try { self::poll($client, 'workflow'); self::fail('The selected receipt must carry its kind.'); } + catch (TaskDeliveryUncertain) {} + self::assertSame(0, $observed); + self::assertSame(self::claimed('workflow'), self::poll(clone $client, 'workflow')); + self::assertSame($transport->requests[2], $transport->requests[3]); + } else { + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + } + self::assertSame(1, $observed); + $polls = array_values(array_filter($recorder->events, static fn (array $event): bool => + $event['name'] === 'durable_workflow_client_requests' && $event['tags']['route'] === '/worker/workflow-tasks/poll')); + self::assertCount($invalidFirst ? 2 : 1, $polls); + self::assertCount($invalidFirst ? 4 : 3, $transport->requests); + self::assertStringNotContainsString('opaque-first-packet', json_encode($recorder->events, JSON_THROW_ON_ERROR)); + } + + /** @return array */ + public static function metricReceipts(): array { return [[false], [true]]; } + + #[DataProvider('kinds')] + public function testOnlyPositiveAcceptanceReturnsAnExecutableTask(string $kind): void + { + $transport = new FakeTransport([self::registration(), self::offer($kind), self::claimed($kind)]); + $observed = []; + $client = (new Client('https://server.example', transport: $transport))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$observed): ?\Closure { + if ($response['task'] ?? null) { + $observed[] = $path; + } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed($kind), self::poll($client, $kind)); + self::assertCount(3, $transport->requests); + self::assertSame(['/worker/'.$kind.'-tasks/poll'], $observed); + $initial = $transport->requests[1]; + $accept = $transport->requests[2]; + self::assertStringEndsWith('/worker/'.$kind.'-tasks/offers', $initial['uri']); + self::assertSame($initial['uri'], $accept['uri']); + self::assertSame(TaskDelivery::VERSION, $initial['headers'][TaskDelivery::HEADER]); + self::assertSame($initial['headers'], $accept['headers']); + self::assertSame(4, $initial['body']['timeout_seconds']); + self::assertSame(array_replace($initial['body'], ['timeout_seconds' => 0, 'task_offer_token' => 'opaque-first-packet']), $accept['body']); + self::assertSame(str_repeat('a', 32), $initial['body']['registration_token']); + self::assertSame(getmypid(), $transport->requests[0]['body']['process_metrics']['process_id']); + self::assertArrayNotHasKey(TaskDelivery::HEADER, $transport->requests[0]['headers']); + } + + #[DataProvider('uncertainOutcomes')] + public function testUncertainRepliesRetryOnlyTheFirstSelectedPacket(TransportException|array|null $outcome): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), $outcome, self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'workflow'); + self::fail('The uncertain acceptance must not authorize execution or start another poll.'); + } catch (TaskDeliveryUncertain $error) { + self::assertTrue(PollResponse::isTransientFailure($error)); + self::assertSame('fixed-poll', $error->details['poll_request_id']); + self::assertArrayNotHasKey('task_offer_token', $error->details); + } + // A clone and changed wait/page inputs cannot replace an unresolved packet. + $clone = $client->withWorkerPollCutoffObserver(static function (): void {}); + self::assertSame(self::claimed('workflow'), $clone->pollWorkflowTaskResponse('worker', 'orders', 19, 'fixed-poll', 30)); + self::assertCount(4, $transport->requests); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertSame('opaque-first-packet', $transport->requests[3]['body']['task_offer_token']); + } + + public static function uncertainOutcomes(): iterable + { + yield 'lost response' => [new TransportException('Connection reset.', transientConnectionFailure: true)]; + yield 'gateway timeout' => [new TransportException('Gateway timeout.', 504)]; + yield 'older node' => [new TransportException('Not found.', 404)]; + yield 'structured backend failure' => [new TransportException('Backend unavailable.', 503, + ['task' => null, 'reason' => 'backend_unavailable', 'retryable' => true])]; + yield 'lock pressure' => [new TransportException('Lock pressure.', 503, + ['task' => null, 'reason' => 'backend_lock_pressure', 'poll_status' => 'backend_lock_pressure', 'retry_after_seconds' => 1])]; + yield 'packet unavailable on another node' => [new TransportException('Unavailable.', 409, + ['task' => null, 'reason' => 'task_offer_unavailable'])]; + yield 'empty is not proof of no claim' => [['task' => null, 'poll_status' => 'empty']]; + yield 'malformed response' => [null]; + yield 'wrong task' => [['task' => ['task_id' => 'other', 'lease_owner' => 'worker']]]; + yield 'wrong owner' => [['task' => ['task_id' => 'task', 'lease_owner' => 'other']]]; + } + + #[DataProvider('overlappingPolls')] + public function testOverlappingPollsHaveOnlyOneAcceptanceOwner(string $kind, bool $clone, string $pauseAt): void + { + $offers = $acceptances = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, $pauseAt, &$offers, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (isset($body['task_offer_token'])) { + if (++$acceptances === 1 && $pauseAt === 'accept') { \Fiber::suspend('accept'); } + return self::claimed($kind); + } + if (++$offers === 1 && $pauseAt === 'offer') { \Fiber::suspend('offer'); } + return self::offer($kind); + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static fn (): array => self::poll($client, $kind)); + self::assertSame($pauseAt, $owner->start()); + $contender = new \Fiber(static fn (): array => self::poll($other, $kind)); + $contender->start(); + self::assertTrue($contender->isTerminated()); + $owner->resume(); + self::assertSame(self::claimed($kind), $owner->getReturn()); + self::assertSame(['task' => null], $contender->getReturn()); + self::assertSame(1, $offers); + self::assertSame(1, $acceptances); + } + + #[DataProvider('overlappingClients')] + public function testUncertainAcceptanceOwnerReleasesTheGateWithoutReplacingItsPacket(string $kind, bool $clone): void + { + $offers = $acceptances = 0; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, &$offers, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (!isset($body['task_offer_token'])) { return self::offer($kind, 'packet-'.++$offers); } + if (++$acceptances === 1) { + \Fiber::suspend('accept'); + throw new TransportException('Gateway timeout.', 504); + } + return self::claimed($kind); + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static function () use ($client, $kind): ?TaskDeliveryUncertain { + try { self::poll($client, $kind); } + catch (TaskDeliveryUncertain $error) { return $error; } + return null; + }); + self::assertSame('accept', $owner->start()); + $contender = new \Fiber(static fn (): array => self::poll($other, $kind)); + $contender->start(); + $owner->resume(); + self::assertInstanceOf(TaskDeliveryUncertain::class, $owner->getReturn()); + self::assertSame(['task' => null], $contender->getReturn()); + self::assertSame(self::claimed($kind), self::poll($other, $kind)); + self::assertSame(1, $offers); + self::assertSame(2, $acceptances); + self::assertSame($transport->requests[2], $transport->requests[3]); + } + + public static function overlappingClients(): iterable + { + foreach (['workflow', 'activity', 'query'] as $kind) { + yield $kind.' same client' => [$kind, false]; + yield $kind.' clone' => [$kind, true]; + } + } + + #[DataProvider('overlappingClients')] + public function testAnAcceptanceOwnerDoesNotBlockAnotherLogicalPoll(string $kind, bool $clone): void + { + $acceptances = 0; + $idField = $kind === 'query' ? 'query_task_id' : 'task_id'; + $transport = new FakeTransport(handler: static function (string $method, string $uri, array $headers, ?array $body) use ($kind, $idField, &$acceptances): array { + if (str_ends_with($uri, '/worker/register')) { return self::registration(); } + if (!isset($body['task_offer_token'])) { + $response = self::offer($kind); + $response['task_offer']['task_id'] = $body['poll_request_id']; + return $response; + } + if (++$acceptances === 1) { \Fiber::suspend('accept'); } + $response = self::claimed($kind); + $response['task'][$idField] = $body['poll_request_id']; + return $response; + }); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + $other = $clone ? clone $client : $client; + $owner = new \Fiber(static fn (): array => self::poll($client, $kind)); + self::assertSame('accept', $owner->start()); + self::assertSame('other-poll', self::poll($other, $kind, 'other-poll')['task'][$idField]); + $owner->resume(); + self::assertSame('fixed-poll', $owner->getReturn()['task'][$idField]); + self::assertSame(2, $acceptances); + } + + public static function overlappingPolls(): iterable + { + foreach (self::overlappingClients() as $name => $arguments) { + foreach (['offer', 'accept'] as $phase) { + yield $name.' during '.$phase => [...$arguments, $phase]; + } + } + } + + public function testAnOlderNodeCannotDowngradeTheNegotiatedEndpoint(): void + { + $transport = new FakeTransport([self::registration(), new TransportException('Not found.', 404), ['task' => null]]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'activity'); + self::fail('Negotiated delivery cannot fall back to a legacy immediate claim.'); + } catch (TaskDeliveryUncertain) { + self::assertSame(['task' => null], self::poll($client, 'activity')); + } + self::assertSame($transport->requests[1], $transport->requests[2]); + self::assertStringEndsWith('/activity-tasks/offers', $transport->requests[2]['uri']); + } + + private static function refusal(string $status, string $reason): TransportException + { + $body = ['task' => null, 'poll_status' => $status, 'reason' => $reason]; + return new TransportException('Synthetic refusal.', 409, $body, + taskDeliveryRefusal: \DurableWorkflow\Transport\TaskDeliveryRefusal::fromJson(409, json_encode($body, JSON_THROW_ON_ERROR))); + } + + public function testAConfirmedExpiryReleasesTheSelectedPacket(): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + self::refusal('task_offer_expired', 'task_offer_expired'), + self::offer('workflow', 'second-packet'), self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame('empty', self::poll($client, 'workflow')['poll_status']); + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[3]['body']); + self::assertSame('second-packet', $transport->requests[4]['body']['task_offer_token']); + } + + public function testRegistrationCannotDowngradeAnUnresolvedAcceptance(): void + { + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Gateway timeout.', 504), ['registered' => true], self::claimed('workflow')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { self::poll($client, 'workflow'); self::fail('Expected an uncertain acceptance.'); } + catch (TaskDeliveryUncertain) {} + try { $client->registerWorker('worker', 'orders', [], []); self::fail('Expected a rejected downgrade.'); } + catch (ServerException $error) { self::assertSame('invalid_task_delivery_capability', $error->reason); } + self::assertSame(self::claimed('workflow'), self::poll($client, 'workflow')); + self::assertSame($transport->requests[2], $transport->requests[4]); + } + + public function testAnAuthoritativeDiscoveryWorkerStopNeedsNoAcceptance(): void + { + $terminal = ['task' => null, 'poll_status' => 'draining', 'reason' => 'worker_draining']; + $transport = new FakeTransport([self::registration(), + new TransportException('Worker is draining.', 409, $terminal)]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + self::assertSame($terminal, self::poll($client, 'workflow')); + self::assertTrue(PollResponse::isTerminal($terminal)); + self::assertCount(2, $transport->requests); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[1]['body']); + } + + public function testReplacementRegistrationDiscardsOnlyTheOldIncarnationsPacket(): void + { + $replacement = self::registration(); + $replacement['registration_token'] = str_repeat('b', 32); + $transport = new FakeTransport([self::registration(), self::offer('activity'), + new TransportException('Gateway timeout.', 504), $replacement, self::offer('activity', 'replacement-packet'), self::claimed('activity')]); + $client = new Client('https://server.example', transport: $transport); + $client->registerWorker('worker', 'orders', [], []); + try { self::poll($client, 'activity'); self::fail('Expected an uncertain acceptance.'); } + catch (TaskDeliveryUncertain) {} + $client->registerWorker('worker', 'orders', [], []); + self::assertSame(self::claimed('activity'), self::poll($client, 'activity')); + self::assertSame(str_repeat('b', 32), $transport->requests[4]['body']['registration_token']); + self::assertArrayNotHasKey('task_offer_token', $transport->requests[4]['body']); + self::assertSame('replacement-packet', $transport->requests[5]['body']['task_offer_token']); + } + + #[DataProvider('legacyRegistrations')] + public function testLegacyDeliveryIsChosenOnlyAtRegistration(array $registration, string $protocol, ?array $capabilities): void + { + $transport = new FakeTransport([$registration, ['task' => null]]); + $client = new Client('https://server.example', transport: $transport, workerProtocolVersion: $protocol); + if ($capabilities === null) { + $client->registerWorker('worker', 'orders', [], []); + } else { + $client->registerWorker('worker', 'orders', [], [], $capabilities); + } + self::poll($client, 'workflow'); + self::assertStringEndsWith('/workflow-tasks/poll', $transport->requests[1]['uri']); + self::assertArrayNotHasKey(TaskDelivery::HEADER, $transport->requests[1]['headers']); + if ($protocol === '1.19') { + self::assertNotContains(TaskDelivery::CAPABILITY, $transport->requests[0]['body']['capabilities']); + } + } + + public static function legacyRegistrations(): iterable + { + yield 'no server feature' => [['registered' => true], '1.20', null]; + yield 'explicit worker opt-out' => [self::registration(), '1.20', ['query_tasks']]; + yield 'older protocol' => [self::registration(), '1.19', null]; + } + + public function testAContradictoryOfferCannotReachPayloadOrExecutionObservers(): void + { + $contradictory = self::claimed('workflow') + ['task_offer' => self::offer('workflow')['task_offer']]; + $transport = new FakeTransport([self::registration(), $contradictory]); + $claims = 0; + $client = (new Client('https://server.example', transport: $transport))->withWorkerPayloadObserver( + static function (string $path, ?array $body, array $response) use (&$claims): ?\Closure { + if ($response['task'] ?? null) { ++$claims; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + try { + self::poll($client, 'workflow'); + self::fail('A contradictory offer cannot authorize a task.'); + } catch (ServerException $error) { + self::assertSame('invalid_task_delivery_response', $error->reason); + } + self::assertSame(0, $claims); + self::assertCount(2, $transport->requests); + } + + public function testTheManagedWorkerRetainsItsLogicalPollAfterAnUncertainAcceptance(): void + { + $now = 0.0; + $retries = []; + $transport = new FakeTransport([self::registration(), self::offer('workflow'), + new TransportException('Not found.', 404), + self::refusal('stale_worker_registration', 'worker_registration_lost_authority')]); + $worker = new Worker(new Client('https://server.example', transport: $transport), 'orders', workerId: 'worker', + clock: static function () use (&$now): float { return $now; }, + sleeper: static function (int $microseconds) use (&$now): void { $now += $microseconds / 1_000_000; }, + transientPollRetryObserver: static function (string $kind, int $attempt, float $delay, ServerException $error) use (&$retries): void { + $retries[] = [$kind, $attempt, $error::class]; + }); + $worker->run(4); + self::assertSame([['workflow', 1, TaskDeliveryUncertain::class]], $retries); + self::assertSame($transport->requests[2], $transport->requests[3]); + self::assertContains(TaskDelivery::CAPABILITY, $transport->requests[0]['body']['capabilities']); + self::assertCount(4, $transport->requests); + } + + public function testOfferTransportUsesTheSameProgressAndLogicalPathForAllKinds(): void + { + foreach (['workflow', 'activity', 'query', 'update-validation'] as $kind) { + $path = '/worker/'.$kind.'-tasks/offers'; + self::assertSame($kind, PollTransport::taskKind('POST', $path)); + self::assertSame('/worker/'.$kind.'-tasks/poll', PollTransport::logicalPath($path)); + self::assertNull(PollTransport::taskKind('GET', $path)); + } + } + + public static function kinds(): array { return [['workflow'], ['activity'], ['query']]; } + + private static function poll(Client $client, string $kind, string $pollId = 'fixed-poll'): array + { + return match ($kind) { + 'workflow' => $client->pollWorkflowTaskResponse('worker', 'orders', 4, $pollId), + 'activity' => $client->pollActivityTaskResponse('worker', 'orders', 4, $pollId), + 'query' => $client->pollQueryTaskResponse('worker', 'orders', 4, $pollId), + }; + } + + private static function offer(string $kind, string $token = 'opaque-first-packet'): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'task', + 'token' => $token, 'expires_at_ms' => 1000, + ]]; + } + + private static function claimed(string $kind): array + { + return ['task' => [($kind === 'query' ? 'query_task_id' : 'task_id') => 'task', 'lease_owner' => 'worker', 'task_kind' => $kind]]; + } + + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'heartbeat_interval_seconds' => 10, + 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, + 'version' => TaskDelivery::VERSION, 'endpoint_suffix' => '/offers', + 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ], 'worker_deregistration_fencing' => ['schema' => 'durable-workflow.v2.worker-deregistration.v1', + 'supported' => true, 'endpoint' => '/worker/registrations/{workerId}/deregister', 'receipt_retention_seconds' => 600]]]; + } +} diff --git a/tests/WorkerTaskDeliveryWireTest.php b/tests/WorkerTaskDeliveryWireTest.php new file mode 100644 index 00000000..3669f9c1 --- /dev/null +++ b/tests/WorkerTaskDeliveryWireTest.php @@ -0,0 +1,209 @@ + $input */ + #[DataProvider('wireCases')] + public function testTheOriginalWireReplyControlsOnlyItsSelectedAcceptance(array $input, string $action, string $endpoint, string $kind, bool $raw, bool $progress = false): void + { + $requests = []; + $http = new GuzzleClient(['handler' => static function (RequestInterface $request) use (&$requests, $input, $raw, $kind, $progress) { + $body = json_decode((string) $request->getBody(), true, flags: JSON_THROW_ON_ERROR); + $requests[] = ['uri' => (string) $request->getUri(), 'headers' => $request->getHeaders(), 'body' => (string) $request->getBody()]; + $index = count($requests); + if ($index === 1) { return Create::promiseFor(self::json(self::registration())); } + if ($index === 2 || ($index === 4 && !isset($body['task_offer_token']) && str_ends_with((string) $request->getUri(), '/offers'))) { + return Create::promiseFor(self::json(self::offer($kind))); + } + if ($index === 3) { return Create::promiseFor(self::reply($input, $raw, $progress)); } + $reply = str_ends_with((string) $request->getUri(), '/poll') ? ['task' => null] : self::claimed($kind); + return Create::promiseFor(self::json($reply)); + }]); + $transport = new Psr18Transport(client: $http); + $client = new Client('https://server.example', transport: self::withExceptionContext($transport, $input)); + $observed = 0; + $client = $client->withWorkerPayloadObserver(static function (string $path, ?array $body, array $response) use (&$observed): ?Closure { + if ($response['task'] ?? null) { ++$observed; } + return null; + }); + $client->registerWorker('worker', 'orders', [], []); + $result = null; + $error = null; + try { $result = self::poll($client, $endpoint, 'original'); } + catch (ServerException $caught) { $error = $caught; } + if ($action === 'deliver_selected_task') { + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame($input['expected_claim'], $result); + self::assertSame(1, $observed); + self::assertCount(3, $requests); + return; + } + self::assertSame(0, $observed, 'No refusal or uncertain outcome may start task execution.'); + if ($action === 'retain_selected_packet') { + self::assertInstanceOf(TaskDeliveryUncertain::class, $error); + self::assertSame(self::claimed($kind), self::poll(clone $client, $endpoint, 'original')); + self::assertSame(1, $observed); + self::assertCount(4, $requests); + self::assertSame($requests[2], $requests[3], 'Retry must preserve the selected packet, kind set and request.'); + return; + } + self::assertNull($error, $error?->getMessage() ?? ''); + self::assertSame($action === 'resolve_selected_packet' ? 'empty' : 'stale_worker_registration', $result['poll_status']); + self::poll($client, $endpoint, 'next'); + self::assertStringEndsWith('/'.$endpoint.'-tasks/'.($action === 'resolve_selected_packet' ? 'offers' : 'poll'), $requests[3]['uri']); + self::assertArrayNotHasKey('task_offer_token', json_decode($requests[3]['body'], true, flags: JSON_THROW_ON_ERROR)); + self::assertCount($action === 'resolve_selected_packet' ? 5 : 4, $requests); + } + + /** @return iterable, string, string, string, bool, bool}> */ + public static function wireCases(): iterable + { + foreach (['wire_cases' => false, 'raw_wire_cases' => true] as $group => $raw) { + foreach (self::fixture()[$group] as $case) { + foreach ([['workflow', 'workflow'], ['workflow', 'update_validation'], ['activity', 'activity'], ['query', 'query'], ['update-validation', 'update_validation']] as [$endpoint, $kind]) { + foreach ($raw ? [false, true] : [false] as $progress) { + yield $case['id'].' / '.$endpoint.' / '.$kind.($progress ? ' / progress' : '') => [$case['input'], $case['expected_action'], $endpoint, $kind, $raw, $progress]; + } + } + } + } + $payload = <<<'JSON' +{"zero":0,"negative":-3,"decimal":0.25,"tiny":1e-3,"huge":2E+4,"bool":true,"nil":null,"nested":[false,{},[]],"unicode":"\uD83D\uDE80","quotes":"a\"b","slash":"a\\b","names":{"re\u0061son":1,"other":2}} +JSON; + $expected = ['zero' => 0, 'negative' => -3, 'decimal' => 0.25, 'tiny' => 0.001, 'huge' => 20000.0, + 'bool' => true, 'nil' => null, 'nested' => [false, [], []], 'unicode' => '🚀', 'quotes' => 'a"b', + 'slash' => 'a\\b', 'names' => ['reason' => 1, 'other' => 2]]; + foreach ([['workflow', 'workflow'], ['workflow', 'update_validation'], ['activity', 'activity'], ['query', 'query'], ['update-validation', 'update_validation']] as [$endpoint, $kind]) { + $claim = self::claimed($kind); + $raw = substr(json_encode($claim, JSON_THROW_ON_ERROR), 0, -2).',"custom_data":'.$payload.'}}'; + $claim['task']['custom_data'] = $expected; + foreach ([false, true] as $progress) { + yield 'valid claim values / '.$endpoint.' / '.$kind.($progress ? ' / progress' : '') => [ + ['http_status' => 200, 'raw_body_utf8' => $raw, 'expected_claim' => $claim], 'deliver_selected_task', $endpoint, $kind, true, $progress, + ]; + } + } + } + + /** @param array $input */ + private static function reply(array $input, bool $raw, bool $progress): Response + { + if (isset($input['transport_failure'])) { throw new TransportException('Synthetic body read failure.'); } + if ($raw) { + $bytes = isset($input['raw_body_hex']) ? hex2bin($input['raw_body_hex']) : $input['raw_body_utf8']; + if ($progress) { + return new Response(200, ['Content-Type' => PollTransport::MEDIA_TYPE], + '{"schema":"durable-workflow.poll-transport.v1","status":'.$input['http_status'].',"headers":{},"body":'.$bytes.'}'); + } + return new Response($input['http_status'], ['Content-Type' => 'application/json'], $bytes); + } + // The real progress decoder also exercises wrong-type logical statuses + // that PSR-7's integer status constructor cannot represent directly. + return new Response(200, ['Content-Type' => PollTransport::MEDIA_TYPE], json_encode([ + 'schema' => 'durable-workflow.poll-transport.v1', 'status' => $input['http_status'], + 'headers' => (object) [], 'body' => $input['decoded_body'], + ], JSON_THROW_ON_ERROR)); + } + + /** @param array $input */ + private static function withExceptionContext(Psr18Transport $transport, array $input): Transport + { + return new class($transport, $input) implements Transport { + /** @param array $input */ + public function __construct(private Psr18Transport $transport, private array $input) {} + + public function send(string $method, string $uri, array $headers, ?array $body = null): ?array + { + try { return $this->transport->send($method, $uri, $headers, $body); } + catch (TransportException $error) { + $input = $this->input; + if (!array_key_exists('exception_http_status', $input) && !array_key_exists('exception_reason', $input)) { throw $error; } + $status = array_key_exists('exception_http_status', $input) ? $input['exception_http_status'] : $error->status; + $reason = array_key_exists('exception_reason', $input) ? $input['exception_reason'] : ($error->response['reason'] ?? null); + try { $context = new ServerException('Synthetic typed exception context.', $status, $reason, $error->response, $error); } + // Invalid types cannot become typed SDK exception fields. + // Treat that construction failure as unreadable, without coercion. + catch (TypeError $typeError) { throw new TransportException('Unreadable exception context.', previous: $typeError); } + throw $context; + } + } + }; + } + + /** @return array */ + private static function fixture(): array { return json_decode((string) file_get_contents(self::FIXTURE), true, flags: JSON_THROW_ON_ERROR); } + + /** @return array */ + private static function poll(Client $client, string $endpoint, string $poll): array + { + return (new ReflectionMethod(Client::class, 'pollTaskResponse'))->invoke($client, '/worker/'.$endpoint.'-tasks/poll', [ + 'worker_id' => 'worker', 'task_queue' => 'orders', 'timeout_seconds' => 4, 'poll_request_id' => $poll, + 'task_kinds' => $endpoint === 'workflow' ? ['workflow', 'update_validation'] : [str_replace('-', '_', $endpoint)], + ]); + } + + /** @param array $body */ + private static function json(array $body): Response { return new Response(200, ['Content-Type' => 'application/json'], json_encode($body, JSON_THROW_ON_ERROR)); } + + /** @return array */ + private static function registration(): array + { + return ['registered' => true, 'registration_token' => str_repeat('a', 32), 'server_capabilities' => ['task_delivery' => [ + 'schema' => TaskDelivery::SCHEMA, 'supported' => true, 'minimum_protocol_version' => '1.20', + 'worker_capability' => TaskDelivery::CAPABILITY, 'header' => TaskDelivery::HEADER, 'version' => TaskDelivery::VERSION, + 'endpoint_suffix' => '/offers', 'acceptance_timeout_seconds' => 2, 'unaccepted_offer_reserves_task' => false, + ]]]; + } + + /** @return array */ + private static function offer(string $kind): array + { + return ['task' => null, 'poll_status' => 'offered', 'task_offer' => [ + 'schema' => TaskDelivery::SCHEMA, 'task_kind' => $kind, 'task_id' => 'selected-task', 'token' => 'synthetic-packet', 'expires_at_ms' => 1000, + ]]; + } + + /** @return array */ + private static function claimed(string $kind): array + { + $field = match ($kind) { 'query' => 'query_task_id', 'update_validation' => 'update_validation_task_id', default => 'task_id' }; + return ['task' => [$field => 'selected-task', 'task_kind' => $kind, 'lease_owner' => 'worker']]; + } +} diff --git a/tests/fixtures/task-delivery-contract/README.md b/tests/fixtures/task-delivery-contract/README.md new file mode 100644 index 00000000..3ac933c2 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/README.md @@ -0,0 +1,42 @@ +# Task delivery source contract + +`offer-accept.v1.json` is byte-identical to Server commit +`955da4e870ef76e7df1bcd24beaae037df63a8f4`, path +`tests/Fixtures/contracts/task-delivery-offers/offer-accept.v1.json`: + +- Git blob: `92df9dd2243521a509bfc8b6ce61a7be16711060` +- SHA-256: `cac2bc6852a09db9526c623c3683772ebbba7d3f1d0b401fb92ae2026cd32b11` +- Size: 18,142 bytes; 49 scenario IDs. + +The source contract is reviewed in Server #1033 for the unimplemented native +#931 capability. Its status remains draft contract data, without an executor +or qualification result. `WorkerTaskDeliveryKindsTest` drives the real PHP +Client/delivery state through an in-memory transport for the five `kind-*` +rows only. Supplementary PHP controls cover malformed kind sets, immutable +selection, clones and uncertainty. They do not execute the other native +claim/receipt, seal/key, database, proxy, restart or capacity scenarios. Reading +or enumerating those rows supplies no execution result. Tokens are synthetic +opaque test text, not sealed native packets or real credentials. + +`acceptance-refusals.v1.json` is byte-identical to Server commit +`31b281ac3891143eae9c6c5ace9c65b63e404e17`, tree +`d0d1cf302f2076c3307d8bc71c8196d6b6d3c062`, path +`tests/Fixtures/contracts/task-delivery-offers/acceptance-refusals.v1.json`: + +- SHA-256: `64a936382d4ce829d0e1de50e75247223711bf4011c830b257143b0a56c237ae` +- Size: 74,169 bytes; 175 unique IDs: 117 decoded wire, 34 raw wire and + 24 authority scenarios. +- Exact companion source review: + https://github.com/durable-workflow/server/issues/931#issuecomment-6659734780 + +`WorkerTaskDeliveryWireTest` sends the 117 decoded wire cases through the actual +PHP progress-envelope transport and the 34 original-byte cases through both +ordinary and progress-envelope responses. Each runs for workflow, mixed-kind +workflow selecting update validation, activity, query and dedicated update +validation. Exception-context cases preserve each present field; fields that +cannot be represented by PHP's strict exception constructor fail as unreadable, +without coercion. Invalid UTF-8 hex is decoded byte-for-byte before transport. +Supplementary client-state controls cover clones, pending-request isolation and +replacement incarnations. These are in-memory PHP source tests. The 24 Server +authority/receipt/race scenarios are inventoried only; neither those requirements +nor native transactions, runtime behavior or installed packages are qualified. diff --git a/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json b/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json new file mode 100644 index 00000000..e466f916 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/acceptance-refusals.v1.json @@ -0,0 +1,2296 @@ +{ + "schema": "durable-workflow.task-delivery-acceptance-refusals.v1", + "status": "source contract data; no classifier, executor, capability or qualification result", + "source_issue": "https://github.com/durable-workflow/server/issues/931", + "decisions": { + "general": "https://github.com/durable-workflow/server/issues/931#issuecomment-6659034006", + "security_design": "https://github.com/durable-workflow/server/issues/931#issuecomment-6659043856" + }, + "parent_fixture": { + "commit": "955da4e870ef76e7df1bcd24beaae037df63a8f4", + "path": "tests/Fixtures/contracts/task-delivery-offers/offer-accept.v1.json", + "sha256": "cac2bc6852a09db9526c623c3683772ebbba7d3f1d0b401fb92ae2026cd32b11", + "bytes": 18142, + "case_count": 49, + "unchanged_required": true + }, + "contract": { + "applies_to": "a selected acceptance packet; ordinary discovery before selection retains its separate poll contract", + "http_status": 409, + "body_type": "JSON object", + "exact_body_keys": [ + "task", + "poll_status", + "reason" + ], + "task": null, + "poll_status_type": "string", + "reason_type": "string", + "allowed_responses": [ + { + "id": "expiry", + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "action": "resolve_selected_packet", + "requires": "authoritative serialized no-acceptance proof; committed live receipt wins over deadline" + }, + { + "id": "authority_loss", + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "action": "end_original_registration", + "requires": "authoritative proof original incarnation can no longer execute; mode/freshness label alone is insufficient" + } + ], + "available_exception_fields_must_match_response": true, + "raw_response_before_ordinary_decode": { + "strict_utf8_json_required": true, + "exactly_one_complete_json_value_required": true, + "only_json_whitespace_may_follow_value": true, + "partial_prefix_parsing_or_trailing_payload_allowed": false, + "duplicate_decoded_member_names_forbidden": true, + "duplicate_identical_values_also_forbidden": true, + "last_member_wins_or_first_member_wins_not_sufficient": true, + "unavailable_raw_uniqueness_proof": "retain_selected_packet", + "unique_escaped_member_names": "ordinary JSON decoding applies; compare decoded distinct names and exact scalar values" + }, + "normalization_or_aliases_allowed": false, + "other_response_action": "retain_selected_packet", + "uncertainty_invariants": { + "original_packet_endpoint_registration_poll_and_kind_set_preserved": true, + "unrelated_pending_packets_preserved": true, + "payload_observation_hydration_execution_authorized": false, + "new_logical_poll_authorized": false, + "legacy_claim_authorized": false, + "end_original_registration_authorized": false + }, + "per_response_scope": "expiry resolves only selected packet; proved authority loss ends only the original registration, never a replacement incarnation" + }, + "source_executor_requirements": { + "raw_case_encoding": "each raw case has exactly one raw_body_utf8 or lowercase raw_body_hex input; hex cases bind raw_body_bytes; each decoded raw vector is at most 512 bytes", + "kinds": [ + "workflow", + "activity", + "query", + "update_validation" + ], + "inputs": "synthetic authorized namespace, registration incarnations, tasks, clocks and faults; no real credentials", + "wire_vectors": "bind every decoded and raw wire case ID to actual SDK selected-acceptance handling before payload observers/hydration; enforce strict duplicate-name rejection on the original response before ordinary lossy decoding", + "authority_vectors": "bind every authority case ID to actual native receipt/claim/registration transaction boundaries on each required backend", + "race_controls": "independent connections with barriers at actual authority locks, commits, rollback and replacement; a scenario-only model is insufficient", + "before_after_observations": [ + "logical poll and selected packet identity", + "current original/replacement incarnation and fence", + "receipt task/kind/owner/attempt", + "current lease authority including renewal", + "attempt counter", + "task-start history", + "unrelated pending logical polls" + ], + "receipt_precedence": "serialize before a negative decision; durable live receipt replays same current claim after offer expiry and restart; fenced/ended claim never replays", + "no_result_inference": "enumerating or validating this source data proves no native transaction, SDK execution, installed capability or runtime result" + }, + "wire_cases": [ + { + "id": "refusal-expiry-exact", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-authority-loss-exact", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-expiry-matching-exception", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": "task_offer_expired" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-authority-matching-exception", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-expiry-matching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409 + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-expiry-matching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": "task_offer_expired" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "refusal-expiry-mismatching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 500 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-mismatching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": "unrelated_terminal_reason" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": true + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": "409" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-200", + "input": { + "http_status": 200, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-400", + "input": { + "http_status": 400, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-401", + "input": { + "http_status": 401, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-403", + "input": { + "http_status": 403, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-404", + "input": { + "http_status": 404, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-405", + "input": { + "http_status": 405, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-408", + "input": { + "http_status": 408, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-410", + "input": { + "http_status": 410, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-422", + "input": { + "http_status": 422, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-429", + "input": { + "http_status": 429, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-500", + "input": { + "http_status": 500, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-wrong-http-503", + "input": { + "http_status": 503, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-task", + "input": { + "http_status": 409, + "decoded_body": { + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-poll-status", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-missing-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-offer-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_offer": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-offer-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-retryable-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "retryable": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-retry-after-ms-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "retry_after_ms": 0 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-task-id-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "task_id": "synthetic-task-A" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-extra-result-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired", + "result": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-cross-pair", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 500, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": "409", + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": 409, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-expiry-exception-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + }, + "exception_http_status": true, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-matching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409 + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-authority-matching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "end_original_registration" + }, + { + "id": "refusal-authority-mismatching-exception-status-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 500 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-mismatching-exception-reason-only", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": "unrelated_terminal_reason" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": true + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": "409" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-only-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-200", + "input": { + "http_status": 200, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-400", + "input": { + "http_status": 400, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-401", + "input": { + "http_status": 401, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-403", + "input": { + "http_status": 403, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-404", + "input": { + "http_status": 404, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-405", + "input": { + "http_status": 405, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-408", + "input": { + "http_status": 408, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-410", + "input": { + "http_status": 410, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-422", + "input": { + "http_status": 422, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-429", + "input": { + "http_status": 429, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-500", + "input": { + "http_status": 500, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-wrong-http-503", + "input": { + "http_status": 503, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-task", + "input": { + "http_status": 409, + "decoded_body": { + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-poll-status", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "reason": "worker_registration_lost_authority" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-missing-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-offer-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_offer": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-offer-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-retryable-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "retryable": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-retry-after-ms-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "retry_after_ms": 0 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-task-id-value", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "task_id": "synthetic-task-A" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-extra-result-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority", + "result": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-cross-pair", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-disagrees", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 500, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-wrong-type", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": "409", + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": 409, + "exception_reason": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-authority-exception-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + }, + "exception_http_status": true, + "exception_reason": "worker_registration_lost_authority" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-string", + "input": { + "http_status": "409", + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-boolean", + "input": { + "http_status": true, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-null", + "input": { + "http_status": null, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-http-status-fraction", + "input": { + "http_status": 409.5, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": null, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": false, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": 409, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": [], + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": {}, + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-null", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": null + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": false + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": 409 + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": [] + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": {} + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-boolean", + "input": { + "http_status": 409, + "decoded_body": { + "task": false, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-integer", + "input": { + "http_status": 409, + "decoded_body": { + "task": 0, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-string", + "input": { + "http_status": 409, + "decoded_body": { + "task": "", + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-array", + "input": { + "http_status": 409, + "decoded_body": { + "task": [], + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-object", + "input": { + "http_status": 409, + "decoded_body": { + "task": {}, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-task-not-null-payload", + "input": { + "http_status": 409, + "decoded_body": { + "task": { + "task_id": "synthetic-task-A" + }, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-null", + "input": { + "http_status": 409, + "decoded_body": null + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-array", + "input": { + "http_status": 409, + "decoded_body": [] + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-boolean", + "input": { + "http_status": 409, + "decoded_body": false + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-string", + "input": { + "http_status": 409, + "decoded_body": "task_offer_expired" + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-body-integer", + "input": { + "http_status": 409, + "decoded_body": 409 + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-draining", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "draining", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stopped", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stopped", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-heartbeat-underscore-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "heartbeat_stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-heartbeat-stale", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "heartbeat-stale", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale-underscore-worker-underscore-registration", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-legacy-stale-reason", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "stale_worker_registration" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-empty-poll", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "empty" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-offered-response", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "offered", + "task_offer": { + "task_id": "synthetic-task-A" + } + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-reason-leading-space", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": " task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-poll-status-case-change", + "input": { + "http_status": 409, + "decoded_body": { + "task": null, + "poll_status": "TASK_OFFER_EXPIRED", + "reason": "task_offer_expired" + } + }, + "expected_action": "retain_selected_packet" + }, + { + "id": "refusal-unreadable-body", + "input": { + "http_status": 409, + "decoded_body": null, + "transport_failure": "body_read_failed" + }, + "expected_action": "retain_selected_packet" + } + ], + "raw_wire_cases": [ + { + "id": "raw-refusal-expiry-exact", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-authority-exact", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "end_original_registration" + }, + { + "id": "raw-refusal-unique-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"re\\u0061son\":\"task_offer_expired\"}" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-expiry-duplicate-task-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-task-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":{\"task_id\":\"synthetic-task-A\"},\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-poll-status-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-poll-status-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"draining\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-reason-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-reason-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"worker_registration_lost_authority\",\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-expiry-duplicate-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"re\\u0061son\":\"worker_registration_lost_authority\",\"reason\":\"task_offer_expired\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-task-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-task-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":{\"task_id\":\"synthetic-task-A\"},\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-poll-status-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"stale_worker_registration\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-poll-status-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"poll_status\":\"draining\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-reason-identical", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"worker_registration_lost_authority\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-reason-conflicting", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"reason\":\"task_offer_expired\",\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-authority-duplicate-escaped-key", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"re\\u0061son\":\"task_offer_expired\",\"reason\":\"worker_registration_lost_authority\"}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "duplicate_decoded_member_name" + }, + { + "id": "raw-refusal-truncated-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "invalid_json" + }, + { + "id": "raw-refusal-expiry-trailing-json-whitespace", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} \r\n\t" + }, + "expected_action": "resolve_selected_packet" + }, + { + "id": "raw-refusal-expiry-trailing-text", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} trailing" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-trailing-second-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} {}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-trailing-second-scalar", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"task_offer_expired\",\"reason\":\"task_offer_expired\"} null" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-expiry-utf8-invalid-leading", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564ff227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-lone-continuation", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f6578706972656480227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-surrogate", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564eda080227d", + "raw_body_bytes": 81 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-expiry-utf8-truncated-multibyte", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227461736b5f6f666665725f65787069726564222c22726561736f6e223a227461736b5f6f666665725f65787069726564e2227d", + "raw_body_bytes": 79 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-trailing-json-whitespace", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} \r\n\t" + }, + "expected_action": "end_original_registration" + }, + { + "id": "raw-refusal-authority-trailing-text", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} trailing" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-trailing-second-object", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} {}" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-trailing-second-scalar", + "input": { + "http_status": 409, + "raw_body_utf8": "{\"task\":null,\"poll_status\":\"stale_worker_registration\",\"reason\":\"worker_registration_lost_authority\"} null" + }, + "expected_action": "retain_selected_packet", + "required_predecode_refusal": "trailing_nonwhitespace" + }, + { + "id": "raw-refusal-authority-utf8-invalid-leading", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479ff227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-lone-continuation", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f7269747980227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-surrogate", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479eda080227d", + "raw_body_bytes": 104 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + }, + { + "id": "raw-refusal-authority-utf8-truncated-multibyte", + "input": { + "http_status": 409, + "raw_body_hex": "7b227461736b223a6e756c6c2c22706f6c6c5f737461747573223a227374616c655f776f726b65725f726567697374726174696f6e222c22726561736f6e223a22776f726b65725f726567697374726174696f6e5f6c6f73745f617574686f72697479e2227d", + "raw_body_bytes": 102 + }, + "required_predecode_refusal": "invalid_utf8", + "expected_action": "retain_selected_packet" + } + ], + "authority_cases": [ + { + "id": "authority-expired-never-accepted", + "before": "original registration current; selected offer is past deadline; no claim or receipt", + "ordered_actions": [ + "take logical-poll/claim authority locks", + "read authoritative receipt and task in serialized state", + "confirm no acceptance can commit for the selected packet", + "emit expiry refusal" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "new_claims": 0, + "new_attempts": 0, + "new_start_events": 0, + "registration_preserved": true + } + }, + { + "id": "authority-acceptance-commits-before-expiry-reader", + "before": "valid acceptance owns authority locks before deadline; expiry reader is concurrent", + "ordered_actions": [ + "acceptance commits receipt, live lease, attempt and start event atomically", + "expiry reader waits for the same authority locks", + "advance past offer deadline", + "expiry reader observes committed live receipt", + "retry original selected packet" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true, + "expiry_refusal_forbidden": true + } + }, + { + "id": "authority-expiry-decision-wins-before-late-acceptance", + "before": "past deadline; no committed receipt; independent acceptance is concurrent", + "ordered_actions": [ + "expiry reader acquires shared transition authority first", + "read authoritative receipt/task and decide expiry under the locks", + "release transaction", + "late acceptance rechecks deadline and authoritative state under the same locks" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "total_new_claims": 0, + "total_new_attempts": 0, + "total_new_start_events": 0, + "acceptance_cannot_commit_after_refusal": true + } + }, + { + "id": "authority-inflight-acceptance-rolls-back", + "before": "acceptance has uncommitted receipt/lease/attempt/start writes; expiry reader waits", + "ordered_actions": [ + "roll back all acceptance writes", + "expiry reader acquires authority locks after rollback", + "observe no committed receipt and expired original packet" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "total_new_claims": 0, + "total_new_attempts": 0, + "total_new_start_events": 0 + } + }, + { + "id": "authority-commit-ack-lost", + "before": "acceptance transaction commits but caller receives a commit/transport error", + "ordered_actions": [ + "retain selected packet on uncertain commit acknowledgement", + "advance past offer deadline", + "retry identical packet through another connection", + "serialize and read authoritative committed receipt/task" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true, + "legacy_claim_requests": 0 + } + }, + { + "id": "authority-lagging-replica-misses-receipt", + "before": "primary has a committed live claim; an older replica cannot see its receipt", + "ordered_actions": [ + "route retry past deadline to lagging replica", + "refuse to infer absence from stale state", + "retain selected packet until authoritative serialized read is possible" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "expiry_refusal_forbidden": true, + "payload_execution_authorized": false + } + }, + { + "id": "authority-receipt-read-fails", + "before": "receipt/task authority read is unavailable or transaction outcome is unknown", + "ordered_actions": [ + "fail authoritative read or serialization", + "do not manufacture a no-claim proof from timeout/error/empty response" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "definitive_refusal_forbidden": true, + "payload_execution_authorized": false + } + }, + { + "id": "authority-replay-after-process-restart", + "before": "acceptance committed a live claim and durable receipt before process exit", + "ordered_actions": [ + "restart accepting process", + "advance past offer deadline while lease remains live", + "retry original packet on a fresh capable process", + "read durable receipt and current authority" + ], + "expected": { + "outcome": "replay_original_live_claim", + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "same_task_owner_attempt": true + } + }, + { + "id": "authority-renewed-live-claim", + "before": "committed claim was renewed beyond its original lease and offer deadline", + "ordered_actions": [ + "advance beyond original lease expiry but before current renewed lease expiry", + "retry identical selected packet", + "read receipt and current lease rather than original token deadline" + ], + "expected": { + "outcome": "replay_original_live_claim", + "same_task_owner_attempt": true, + "retry_new_attempts": 0, + "retry_new_start_events": 0 + } + }, + { + "id": "authority-live-receipt-premature-prune", + "before": "a live replayable claim exists but its receipt has incorrectly disappeared", + "ordered_actions": [ + "detect inability to establish required durable live-claim receipt invariant", + "do not translate missing receipt into an expired-unaccepted claim" + ], + "expected": { + "outcome": "invariant_failure_uncertain", + "selected_packet_preserved": true, + "expiry_refusal_forbidden": true, + "implementation_qualification_must_fail": true + } + }, + { + "id": "authority-draining-retains-live-claim", + "before": "original registration and committed claim remain authoritative; worker mode becomes draining", + "ordered_actions": [ + "retry selected packet past offer deadline", + "read committed live receipt before considering worker mode" + ], + "expected": { + "outcome": "replay_original_live_claim", + "same_task_owner_attempt": true, + "mode_alone_cannot_erase_receipt": true, + "authority_loss_refusal_forbidden": true, + "retry_new_attempts": 0, + "retry_new_start_events": 0 + } + }, + { + "id": "authority-draining-no-proof", + "before": "only a draining lifecycle label is known; actual original claim/registration authority has not been proved lost", + "ordered_actions": [ + "receive a draining label while an acceptance is selected", + "preserve original packet and await one of the exact proved outcomes" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false, + "payload_execution_authorized": false + } + }, + { + "id": "authority-stopped-label-no-proof", + "before": "only a stopped lifecycle label is known; no authoritative ended-identity proof", + "ordered_actions": [ + "inspect lifecycle label without replacing authoritative receipt/registration read" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-heartbeat-label-no-proof", + "before": "heartbeat is stale or freshness lookup is unavailable; authority revocation is unproved", + "ordered_actions": [ + "do not equate heartbeat age with an authoritative execution fence" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-replaced-registration-fenced", + "before": "replacement incarnation has authoritatively fenced original registration and all its execution authority", + "ordered_actions": [ + "serialize replacement and original-packet acceptance under normal transition authority", + "read original identity/fence and prove original worker can no longer execute", + "emit authority-loss refusal" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0, + "replacement_authority_unchanged": true + } + }, + { + "id": "authority-ended-registration-proved", + "before": "original registration ended and authoritative task/lease state proves it cannot execute", + "ordered_actions": [ + "read original registration identity plus current claim/lease authority under transition locks", + "prove actual authority loss rather than infer it from mode text" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0 + } + }, + { + "id": "authority-same-second-replacement", + "before": "same worker ID and process metadata recur within one stored timestamp second; original incarnation differs", + "ordered_actions": [ + "use durable incarnation/credential/fence identity rather than timestamp or worker ID alone", + "prove original execution authority revoked" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "new_claims_from_old_packet": 0, + "replacement_authority_unchanged": true + } + }, + { + "id": "authority-registration-row-unavailable", + "before": "original registration lookup is missing through incomplete/failed observation, not proved authoritative absence", + "ordered_actions": [ + "refuse authority-loss conclusion without a complete authoritative identity/fence read" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "end_original_registration": false + } + }, + { + "id": "authority-selected-claim-ended-registration-current", + "before": "selected claim has ended or its task attempt changed, but original registration still has execution authority for other work", + "ordered_actions": [ + "forbid replay of ended/replaced selected claim", + "do not infer registration-wide authority loss from this task alone", + "retain original packet without inventing a third definitive refusal pair" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "old_payload_replayed": false, + "end_original_registration": false, + "other_claim_authority_unchanged": true + } + }, + { + "id": "authority-replacement-wins-race", + "before": "replacement and acceptance race on independent connections for the original incarnation", + "ordered_actions": [ + "replacement acquires authority locks first", + "replacement commits fence/ended original execution authority", + "old acceptance then reads current registration and task authority" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "total_new_claims_from_old_packet": 0, + "total_new_attempts_from_old_packet": 0, + "total_new_start_events_from_old_packet": 0 + } + }, + { + "id": "authority-acceptance-wins-then-replacement", + "before": "valid acceptance and replacement race on independent connections", + "ordered_actions": [ + "acceptance acquires authority locks first and atomically commits one claim/receipt/attempt/start", + "replacement then serializes and revokes original execution authority", + "retry original selected packet after replacement", + "read receipt but also current authority; never replay a fenced claim" + ], + "expected": { + "outcome": "authority_loss_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "stale_worker_registration", + "reason": "worker_registration_lost_authority" + } + }, + "total_new_claims": 1, + "retry_new_attempts": 0, + "retry_new_start_events": 0, + "old_payload_replayed_after_fence": false + } + }, + { + "id": "authority-expired-offer-live-registration-other-claim", + "before": "selected offer never accepted and is expired; task is now leased by a peer; original registration remains current", + "ordered_actions": [ + "serialize receipt/task read and prove no selected-packet acceptance can commit", + "preserve peer lease and original registration" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "new_claims_from_selected_packet": 0, + "peer_authority_unchanged": true, + "registration_preserved": true + } + }, + { + "id": "authority-refusal-preserves-unrelated-polls", + "before": "two distinct logical polls are pending on the same registration; only selected poll A is proved expired", + "ordered_actions": [ + "receive exact expiry refusal for A", + "resolve only A", + "preserve pending original packet and identity for B" + ], + "expected": { + "outcome": "expiry_refusal", + "response": { + "http_status": 409, + "body": { + "task": null, + "poll_status": "task_offer_expired", + "reason": "task_offer_expired" + } + }, + "poll_b_packet_unchanged": true, + "registration_preserved": true + } + }, + { + "id": "authority-inconsistent-envelope-preserves-all", + "before": "selected poll A and unrelated B are pending; transport/exception/body disagree", + "ordered_actions": [ + "receive mixed or extra-key refusal candidate", + "preserve A and B", + "suppress payload observer, hydration and execution", + "do not begin a replacement logical poll or legacy claim" + ], + "expected": { + "outcome": "uncertain", + "selected_packet_preserved": true, + "poll_b_packet_unchanged": true, + "end_original_registration": false, + "new_logical_poll_requests": 0, + "legacy_claim_requests": 0 + } + } + ] +} diff --git a/tests/fixtures/task-delivery-contract/offer-accept.v1.json b/tests/fixtures/task-delivery-contract/offer-accept.v1.json new file mode 100644 index 00000000..92df9dd2 --- /dev/null +++ b/tests/fixtures/task-delivery-contract/offer-accept.v1.json @@ -0,0 +1,355 @@ +{ + "schema": "durable-workflow.task-delivery-scenarios.v1", + "status": "draft contract data; no executor or qualification result", + "source_issue": "https://github.com/durable-workflow/server/issues/931", + "reference_sources": { + "retired_server_php_commit": "ed302326f0f73c383df88a918c8378755e0a56fe", + "held_sdk_php_commit": "034c0f4dc2fbbc1671fe9b1cdfd0fa36c2576c2e" + }, + "contract": { + "schema": "durable-workflow.task-delivery.v1", + "version": "offer-accept.v1", + "minimum_protocol_version": "1.20", + "worker_capability": "task_delivery_offers", + "header": "X-Durable-Workflow-Task-Delivery", + "endpoint_suffix": "/offers", + "acceptance_timeout_seconds": { + "default": 2, + "minimum": 1, + "maximum": 10, + "actual_integer_required": true + }, + "offer_token_maximum_bytes": 8192, + "unaccepted_offer_reserves_task": false, + "endpoints": { + "workflow": ["workflow", "update_validation"], + "activity": ["activity"], + "query": ["query"], + "update-validation": ["update_validation"] + } + }, + "executor_requirements": { + "inputs": "synthetic authorized registration, tasks, clocks and faults; no real credentials", + "packets": "seal synthetic authoritative contexts with the reviewed implementation; do not use placeholder tokens", + "database": "exercise actual claim/receipt transactions, not a scenario-only model", + "comparison": "compare task authority and attempts, start history and receipt before and after every phase", + "scope": "run on each source backend required by the owning PR; installed proxy/restart/capacity acceptance is separate", + "no_result_inference": "parsing or enumerating this data is not a passing execution" + }, + "security_source_direction": { + "review": "https://github.com/durable-workflow/server/issues/931#issuecomment-6646286980", + "status": "design direction accepted; implementation acceptance held", + "algorithm": "full-round-XChaCha20Poly1305", + "key_bytes": 32, + "nonce_bytes": 24, + "tag_bytes": 16, + "maximum_plaintext_bytes": 6000, + "decoded_frame_bytes": {"minimum": 40, "maximum": 6040}, + "maximum_frame_bytes_with_32_byte_key_id": 8092, + "requirements": ["exact-reviewed-locked-graph", "supported-zeroization-feature", "fallible-os-rng", "closed-canonical-parsers", "persisted-key-id-references", "enforceable-rotation-and-startup-preflight"], + "key_ring_proposal": { + "setting": "DW_TASK_DELIVERY_SEAL_KEYS", + "maximum_raw_utf8_bytes": 4096, + "absent": "unsupported", + "supplied_empty": "configuration-refusal", + "container_fields": ["schema", "keys"], + "schema": "durable-workflow.task-delivery-seal-keys.v1", + "minimum_entries": 1, + "maximum_entries": 4, + "entry_fields": ["id", "key_base64"], + "key_id_ascii_pattern": "[A-Za-z0-9_-]{1,32}", + "key_text_encoding": "RFC4648-standard-padded-canonical", + "key_text_ascii_bytes": 44, + "key_text_trailing_padding_bytes": 1, + "canonical_reencode_required": true, + "validate_all_original_key_text_before_any_decode_or_selection": true, + "validate_container_formatting_as_credential_text": false + } + }, + "cases": [ + { + "id": "offer-lost", + "kinds": ["workflow", "activity", "query", "update_validation"], + "actions": ["observe-ready-task", "drop-offer-response", "peer-claim"], + "expected": {"offer_started_attempts": 0, "offer_start_events": 0, "peer_claims_same_task": true} + }, + { + "id": "offer-peer-wins", + "actions": ["observe-ready-task", "peer-claims-task", "accept-original-offer"], + "expected": {"new_claims_from_accept": 0, "peer_authority_unchanged": true} + }, + { + "id": "offer-expired-unaccepted", + "actions": ["observe-ready-task", "advance-past-offer-expiry", "accept-original-offer", "start-new-logical-poll"], + "expected": {"refusal_status": 409, "reason": "task_offer_expired", "task": null, "old_accept_started_attempts": 0, "new_poll_eligible": true} + }, + { + "id": "accept-lost", + "actions": ["observe-ready-task", "accept-and-commit", "drop-accept-response", "retry-identical-packet"], + "expected": {"total_new_claims": 1, "same_task_owner_attempt": true, "new_start_events_on_retry": 0} + }, + { + "id": "accept-after-offer-expiry", + "actions": ["observe-ready-task", "accept-and-commit", "advance-past-offer-expiry-with-live-lease", "retry-identical-packet"], + "expected": {"replay_accepted_claim": true, "total_new_claims": 1, "new_start_events_on_retry": 0} + }, + { + "id": "accept-concurrent-same", + "actions": ["observe-ready-task", "accept-same-packet-concurrently-through-independent-connections"], + "expected": {"total_new_claims": 1, "all_delivered_claims_same_task_owner_attempt": true} + }, + { + "id": "accept-alternative-offers", + "actions": ["observe-two-eligible-tasks-under-same-logical-poll", "accept-alternative-packets-concurrently"], + "expected": {"maximum_committed_claims": 1, "unselected_task_claimable_by_peer": true, "no_other_task_substituted_for_selected_packet": true} + }, + { + "id": "accept-after-lease-renewal", + "actions": ["accept-and-commit", "renew-same-claim", "advance-past-original-lease-expiry-before-renewed-expiry", "retry-identical-packet"], + "expected": {"replay_current_live_claim": true, "same_task_owner_attempt": true, "new_start_events_on_retry": 0} + }, + { + "id": "accept-after-task-replacement", + "actions": ["accept-and-commit", "end-claim-and-replace-attempt", "retry-original-packet"], + "expected": {"old_claim_delivered": false, "new_claims_from_retry": 0, "replacement_authority_unchanged": true} + }, + { + "id": "registration-replaced", + "actions": ["observe-ready-task", "register-replacement-worker", "accept-old-packet"], + "expected": {"refusal_status": 409, "reason": "worker_registration_lost_authority", "task": null, "new_claims_from_accept": 0} + }, + { + "id": "same-second-replaced", + "actions": ["observe-ready-task", "register-same-worker-id-and-process-metadata-within-same-stored-second", "accept-old-packet"], + "expected": {"refusal_status": 409, "reason": "worker_registration_lost_authority", "task": null, "new_claims_from_accept": 0} + }, + { + "id": "routing-changed", + "actions": ["observe-ready-task", "change-registration-routing-fence", "accept-old-packet"], + "expected": {"new_claims_from_accept": 0, "old_packet_has_authority": false} + }, + { + "id": "cross-namespace", + "actions": ["observe-ready-task", "submit-packet-in-another-authorized-namespace"], + "expected": {"new_claims_from_accept": 0, "neither_namespace_task_authority_changed": true} + }, + { + "id": "changed-packet", + "actions": ["observe-ready-task", "change-one-sealed-byte", "accept-mutated-packet"], + "expected": {"new_claims_from_accept": 0, "seal_rejected_before_claim": true} + }, + { + "id": "kind-workflow", + "endpoint": "workflow", + "requested_kinds": ["workflow"], + "offered_kind": "workflow", + "expected": {"selected_kind_allowed": true, "accepted_kind_must_equal": "workflow"} + }, + { + "id": "kind-validator", + "endpoint": "workflow", + "requested_kinds": ["update_validation"], + "offered_kind": "update_validation", + "expected": {"selected_kind_allowed": true, "accepted_kind_must_equal": "update_validation"} + }, + { + "id": "kind-mixed-order", + "endpoint": "workflow", + "request_kind_orders": [["workflow", "update_validation"], ["update_validation", "workflow"]], + "offered_kind": "update_validation", + "expected": {"same_normalized_kind_set": true, "selected_kind_allowed": true, "accepted_kind_must_equal": "update_validation"} + }, + { + "id": "kind-unrequested", + "endpoint": "workflow", + "requested_kinds": ["workflow"], + "offered_kind": "update_validation", + "expected": {"selected_kind_allowed": false, "accept_requests_sent": 0} + }, + { + "id": "kind-accepted-substitution", + "endpoint": "workflow", + "requested_kinds": ["workflow", "update_validation"], + "selected_kind": "update_validation", + "accepted_kind": "workflow", + "expected": {"execution_authorized": false, "discard_uncertain_selected_packet": false} + }, + { + "id": "accept-db-precommit-fault", + "actions": ["observe-ready-task", "fault-after-claim-writes-before-commit", "inspect-authoritative-storage"], + "expected": {"new_claims_committed": 0, "new_attempts_committed": 0, "new_start_events_committed": 0, "accepted_receipts_committed": 0} + }, + { + "id": "accept-db-uncertain-commit", + "actions": ["observe-ready-task", "commit-acceptance", "lose-database-commit-response", "retry-identical-packet-through-new-connection"], + "expected": {"total_new_claims": 1, "same_task_owner_attempt": true, "legacy_poll_requests": 0} + }, + { + "id": "mixed-old-node", + "actions": ["accept-and-commit", "drop-accept-response", "route-identical-retry-to-old-node-404-or-405", "retry-identical-packet-on-capable-node"], + "expected": {"retain_selected_packet_after_old_node": true, "legacy_poll_requests": 0, "total_new_claims": 1} + }, + { + "id": "proxy-short-wait", + "scope": "separately admitted installed acceptance", + "actions": ["cut-offer-poll-before-configured-wait", "start-task-in-abandoned-poll-window", "observe-live-worker-delivery-and-warning-diagnostics"], + "expected": {"abandoned_task_leases": 0, "wait_capacity_starvation": false, "observed_cutoff_and_configured_wait_reported": true} + }, + { + "id": "restart-wait-capacity", + "scope": "separately admitted installed acceptance", + "actions": ["hold-wait-capacity", "abruptly-restart-server-process", "resume-api-with-same-durable-state", "admit-live-waiter"], + "expected": {"dead_process_owns_wait_slots": false, "task_authority_derived_from_capacity_hint": false} + }, + { + "id": "worker-replacement-capacity", + "scope": "separately admitted installed acceptance", + "actions": ["hold-old-worker-poll", "replace-worker-registration", "release-old-owner-wait-capacity", "poll-as-replacement"], + "expected": {"old_worker_can_claim": false, "replacement_eligible": true, "global_and_namespace_limits_preserved": true} + }, + { + "id": "other-replica-live-waiter", + "scope": "separately admitted installed acceptance", + "actions": ["hold-live-waiter-on-peer-replica", "clean-up-dead-local-owner"], + "expected": {"peer_live_waiter_preserved": true, "foreign_owner_slot_released": false, "global_and_namespace_limits_preserved": true} + }, + { + "id": "nonce-rng-failure", + "scope": "required source controls; not executed", + "actions": ["inject-fallible-os-rng-error", "request-offer"], + "expected": {"packets_emitted": 0, "typed_refusal": true} + }, + { + "id": "nonce-reuse-injection", + "scope": "required source controls; not executed", + "actions": ["inject-repeated-test-nonce", "exercise-seal-wrapper-countercontrol"], + "expected": {"reuse_exposes_refusal_or_failing_countercontrol": true, "production_caller_nonce_accepted": false} + }, + { + "id": "frame-canonical-encoding", + "scope": "required source controls; not executed", + "actions": ["mutate-prefix-case-segments-padding-whitespace-or-base64-alphabet"], + "expected": {"mutated_frames_accepted": 0, "new_claims": 0} + }, + { + "id": "frame-size-boundaries", + "scope": "required source controls; not executed", + "actions": ["exercise-outer-8192-decoded-40-and-6040-and-plaintext-6000-boundaries"], + "expected": {"over_limit_allocation_or_decryption": false, "maximum_32_byte_key_id_frame_bytes": 8092} + }, + { + "id": "aad-domain-ambiguity", + "scope": "required source controls; not executed", + "actions": ["change-prefix-schema-version-or-key-id-in-structured-aad"], + "expected": {"packets_authenticated": 0, "alternate_plaintext_parser_used": false} + }, + { + "id": "plaintext-closed-schema", + "scope": "required source controls; not executed", + "actions": ["authenticate-test-plaintext-with-duplicate-unknown-missing-or-wrong-type-field"], + "expected": {"new_claims": 0, "alternate_parser_used": false} + }, + { + "id": "keyring-closed-schema", + "scope": "required source controls; not executed", + "actions": ["supply-oversized-container-or-duplicate-unknown-entry-field"], + "expected": {"issuance_supported": false, "content_in_diagnostic": false} + }, + { + "id": "keyring-canonical-key-encoding", + "scope": "required source controls; not executed", + "actions": ["supply-alternate-padding-alphabet-or-noncanonical-key-base64"], + "expected": {"configured_keys_accepted": 0, "decoded_entropy_text_scanned": false} + }, + { + "id": "keyring-duplicate-ids-and-limit", + "scope": "required source controls; not executed", + "actions": ["supply-duplicate-id-or-fifth-entry"], + "expected": {"configuration_accepted": false, "existing_key_evicted": false} + }, + { + "id": "keyring-readonly-key-original-text", + "scope": "required source controls; not executed", + "actions": ["supply-malformed-original-text-in-nonissuing-key"], + "expected": {"configuration_accepted": false, "role_selection_before_validation": false} + }, + { + "id": "keyring-zeroization-lockgraph", + "scope": "required source controls; not executed", + "actions": ["inspect-exact-locked-graph-feature-and-key-buffer-lifecycle-controls"], + "expected": {"reduced_round_feature_enabled": false, "supported_zeroization_enabled": true, "revealing_debug_or_serialization": false} + }, + { + "id": "read-key-missing-live-startup", + "scope": "required source controls; not executed", + "actions": ["accept-and-commit", "restart-with-required-read-key-missing"], + "expected": {"issuance_supported": false, "legacy_fallback": false, "claim_silently_stranded": false} + }, + { + "id": "read-key-missing-live-reload", + "scope": "required source controls; not executed", + "actions": ["accept-and-commit", "reload-with-required-read-key-missing"], + "expected": {"issuance_supported": false, "key_automatically_evicted": false} + }, + { + "id": "rotation-renewal-replay", + "scope": "required source controls; not executed", + "actions": ["accept-with-old-key", "stop-issuing-old-key", "renew-claim", "attempt-old-read-key-removal", "replay-original-packet"], + "expected": {"required_read_key_retained": true, "same_task_owner_attempt": true} + }, + { + "id": "rotation-unaccepted-grace", + "scope": "required source controls; not executed", + "actions": ["stop-issuing-retiring-key", "attempt-removal-before-maximum-offer-lifetime-plus-admitted-skew"], + "expected": {"required_read_key_retained": true, "undocumented_clock_assumption": false} + }, + { + "id": "keyslot-exhaustion", + "scope": "required source controls; not executed", + "actions": ["retain-four-required-read-keys", "request-fifth-issuing-key"], + "expected": {"configuration_accepted": false, "required_key_evicted": false} + }, + { + "id": "no-support-before-qualification", + "scope": "required source controls; not executed", + "actions": ["check-support-before-complete-ring-durable-key-reference-migration-and-backend-qualification"], + "expected": {"supported_advertised": false} + }, + { + "id": "content-free-refusal", + "scope": "required source controls; not executed", + "actions": ["exercise-authentication-decoding-key-lookup-load-and-rng-errors"], + "expected": {"token_key_plaintext_digest_or_authority_values_in_diagnostics": false} + }, + { + "id": "keyring-presence-and-raw-size", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["omit-setting", "supply-empty-setting", "supply-invalid-utf8", "supply-valid-container-at-4096-bytes", "supply-container-at-4097-bytes"], + "expected": {"absence_supported": false, "empty_or_invalid_utf8_accepted": false, "raw_limit_checked_before_parse": true, "maximum_raw_utf8_bytes": 4096} + }, + { + "id": "keyring-container-version-and-shape", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["supply-wrong-or-missing-schema", "supply-duplicate-or-unknown-container-field", "supply-unordered-object-instead-of-keys-array", "supply-empty-array-or-wrong-entry-type"], + "expected": {"configuration_accepted": false, "alternate_or_legacy_parser_used": false} + }, + { + "id": "keyring-id-byte-boundaries", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["exercise-valid-ascii-id-at-1-and-32-bytes", "exercise-empty-33-byte-or-nonascii-id", "exercise-case-distinct-ids"], + "expected": {"valid_id_byte_range": [1, 32], "invalid_id_accepted": false, "id_case_folded": false} + }, + { + "id": "keyring-key-text-unused-bits", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["supply-44-byte-standard-base64-with-nonzero-unused-bits", "supply-url-safe-substitution-or-wrong-padding", "supply-canonical-32-byte-synthetic-key"], + "expected": {"noncanonical_key_accepted": false, "canonical_key_bytes": 32, "canonical_text_bytes": 44, "canonical_padding_bytes": 1} + }, + { + "id": "keyring-ordered-two-key-read", + "scope": "proposed canonical configuration controls; not executed", + "actions": ["configure-two-distinct-valid-synthetic-keys", "issue-packet", "read-test-packets-under-either-retained-id"], + "expected": {"issuing_key_index": 0, "both_read_keys_usable": true, "every_original_text_validated_before_selection": true, "decoded_entropy_text_scanned": false} + } + ] +}