From 1fc8f3d077c2144530174d5f5587843227d40923 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:00:04 +0000 Subject: [PATCH 01/19] Bound failure reports while preserving original claim semantics --- scripts/qualify-failure-reports.php | 46 +++++ src/Client.php | 25 +++ src/Exception/FailureReportRefused.php | 15 ++ src/Failure/FailureReportLimiter.php | 203 ++++++++++++++++++++ src/Failure/FailureReportLimits.php | 22 +++ src/Failure/FailureText.php | 44 +++++ src/Transport/JsonDocument.php | 20 ++ src/Transport/Psr18Transport.php | 3 +- src/Transport/RequestSizeLimitTransport.php | 11 ++ src/Transport/RuntimePayloadUploads.php | 3 +- src/Worker.php | 15 +- tests/FailureReportTest.php | 192 ++++++++++++++++++ 12 files changed, 585 insertions(+), 14 deletions(-) create mode 100644 scripts/qualify-failure-reports.php create mode 100644 src/Exception/FailureReportRefused.php create mode 100644 src/Failure/FailureReportLimiter.php create mode 100644 src/Failure/FailureReportLimits.php create mode 100644 src/Failure/FailureText.php create mode 100644 src/Transport/JsonDocument.php create mode 100644 src/Transport/RequestSizeLimitTransport.php create mode 100644 tests/FailureReportTest.php diff --git a/scripts/qualify-failure-reports.php b/scripts/qualify-failure-reports.php new file mode 100644 index 00000000..a83363eb --- /dev/null +++ b/scripts/qualify-failure-reports.php @@ -0,0 +1,46 @@ +requests[] = $request; + return new GuzzleHttp\Psr7\Response(200, ['Content-Type' => 'application/json'], '{}'); + } +}; +$client = new DurableWorkflow\Client('https://recording.invalid', + transport: new DurableWorkflow\Transport\Psr18Transport(client: $http)); +$large = str_repeat('m', 3 * 1024 * 1024); +$results = []; +foreach (['activity', 'workflow-task'] as $kind) { + $http->requests = []; + if ($kind === 'activity') { + $client->failActivityTask('original-task', 'original-attempt', 'original-owner', $large, 'OriginalFailure', true); + } else { + $client->failWorkflowTask('original-task', 'original-owner', 3, $large, 'OriginalFailure'); + } + $request = $http->requests[array_key_last($http->requests)]; + $wire = (string) $request->getBody(); + $body = json_decode($wire, true, flags: JSON_THROW_ON_ERROR); + $failure = $body['failure']; + $results[$kind] = ['original_message_bytes' => strlen($large), 'encoded_request_bytes' => strlen($wire), + 'message_bytes' => strlen($failure['message']), 'original_type' => $failure['type'] === 'OriginalFailure', + 'exact_identity' => $body['lease_owner'] === 'original-owner' + && ($kind === 'activity' ? $body['activity_attempt_id'] === 'original-attempt' : $body['workflow_task_attempt'] === 3), + 'marked_message' => str_contains($failure['message'], '[truncated failure report;'), + 'non_retryable' => $failure['non_retryable'] ?? null]; +} +$pass = true; +foreach ($results as $result) { + $pass = $pass && $result['original_type'] && $result['exact_identity'] + && ($baseline ? $result['encoded_request_bytes'] > 2097152 && !$result['marked_message'] + : $result['encoded_request_bytes'] <= 1048576 && $result['marked_message']); +} +echo json_encode(['baseline' => $baseline, 'sdk_version' => Composer\InstalledVersions::getPrettyVersion('durable-workflow/sdk'), + 'pass' => $pass, 'results' => $results], JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT)."\n"; +exit($pass ? 0 : 1); diff --git a/src/Client.php b/src/Client.php index 26d58968..b3f9e0b7 100644 --- a/src/Client.php +++ b/src/Client.php @@ -9,6 +9,9 @@ use DurableWorkflow\Codec\AvroPayloadCodec; use DurableWorkflow\Codec\PayloadCodec; use DurableWorkflow\Exception\CodecException; +use DurableWorkflow\Exception\FailureReportRefused; +use DurableWorkflow\Failure\FailureReportLimiter; +use DurableWorkflow\Failure\FailureReportLimits; use DurableWorkflow\Exception\QueryFailed; use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Exception\SignalFailed; @@ -74,6 +77,7 @@ final class Client implements WorkflowClientInterface private readonly Transport $transport; private readonly PayloadCodec $codec; private readonly RuntimePayloadUploads $payloadUploads; + private FailureReportLimiter $failureReports; private bool $boundedWorkerRequests = false; private ActivityLeaseBinding $activityLeaseBinding; private ?string $activityLeaseBootstrap = null; @@ -134,9 +138,19 @@ public function __construct( $this->transport = $transport ?? new Psr18Transport(requestTimeoutSeconds: $requestTimeoutSeconds); $this->codec = $codec ?? new AvroPayloadCodec(); $this->payloadUploads = new RuntimePayloadUploads($this->transport, $this->baseUri); + $this->failureReports = new FailureReportLimiter(new FailureReportLimits(), $this->transport); $this->activityLeaseBinding = new ActivityLeaseBinding(); } + /** Narrow the encoded failure-report request budget for an application transport. */ + public function withFailureReportLimit(int $requestBytes): self + { + $copy = clone $this; + $copy->failureReports = new FailureReportLimiter(new FailureReportLimits($requestBytes), $this->transport); + + return $copy; + } + public function payloadCodec(): PayloadCodec { return $this->codec; @@ -2134,6 +2148,7 @@ public function failWorkflowTask( string $failureType = 'PhpWorkflowTaskFailure', ?string $reason = null, ?int $sequence = null, + ?string $stackTrace = null, ): array { return $this->worker('POST', '/worker/workflow-tasks/'.$this->segment($taskId).'/fail', [ 'lease_owner' => $leaseOwner, @@ -2143,6 +2158,7 @@ public function failWorkflowTask( 'type' => $failureType, 'reason' => $reason, 'sequence' => $sequence, + 'stack_trace' => $stackTrace, ], static fn (mixed $value): bool => $value !== null), ]); } @@ -2207,6 +2223,8 @@ public function failActivityTask( string $failureType = 'PhpActivityFailure', bool $nonRetryable = false, ?array $details = null, + ?string $stackTrace = null, + ?array $diagnostics = null, ): array { return $this->worker('POST', self::ACTIVITY_TASK_PATH.$this->segment($taskId).'/fail', [ 'activity_attempt_id' => $activityAttemptId, @@ -2216,6 +2234,8 @@ public function failActivityTask( 'type' => $failureType, 'non_retryable' => $nonRetryable, 'details' => $details !== null ? $this->codec->envelope($details) : null, + 'stack_trace' => $stackTrace, + 'diagnostics' => $diagnostics, ]), ]); } @@ -2619,7 +2639,9 @@ private function request( $pollSeconds = $poll ? max(0, min(60, $body['timeout_seconds'])) : 0; $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; if ($body !== null) { + $body = $this->failureReports->request($path, $body); $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget); + $body = $this->failureReports->request($path, $body); } $activityLease = $worker ? $this->activityLeaseBinding->current() : null; if ($activityLease !== null && $activityLease->governs($path)) { @@ -2659,6 +2681,9 @@ private function request( return $result; } catch (TransportException $exception) { + if ($exception instanceof FailureReportRefused) { + throw $exception; + } $details = $exception->response; $reason = is_array($details) && isset($details['reason']) ? (string) $details['reason'] : null; $status = $exception->status ?? 0; diff --git a/src/Exception/FailureReportRefused.php b/src/Exception/FailureReportRefused.php new file mode 100644 index 00000000..9b97598b --- /dev/null +++ b/src/Exception/FailureReportRefused.php @@ -0,0 +1,15 @@ + 'failure_report_refused', 'limit_bytes' => $limit], $previous); + } +} diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php new file mode 100644 index 00000000..e30ecff8 --- /dev/null +++ b/src/Failure/FailureReportLimiter.php @@ -0,0 +1,203 @@ + $body + * @return array + */ + public function request(string $path, array $body): array + { + $endpoint = preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path, $matches) === 1 + && is_array($body['failure'] ?? null); + $commands = preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 + && is_array($body['commands'] ?? null) + && array_filter($body['commands'], static fn (mixed $command): bool => is_array($command) + && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) !== []; + if (!$endpoint && !$commands) { + return $body; + } + $limit = $this->limits->requestBytes; + if ($this->transport instanceof RequestSizeLimitTransport) { + try { + $known = $this->transport->maximumRequestBytes(); + } catch (\Throwable $error) { + throw new FailureReportRefused($limit, $error); + } + if ($known !== null) { + if ($known < 1) { + throw new FailureReportRefused($limit); + } + $limit = min($limit, $known); + } + } + $messageLimit = FailureReportLimits::MESSAGE_BYTES; + $stackLimit = FailureReportLimits::STACK_BYTES; + $dropMetadata = false; + for ($pass = 0; $pass < 16; ++$pass) { + $candidate = $body; + if ($endpoint) { + $candidate['failure'] = $this->report($body['failure'], $messageLimit, $stackLimit, + $dropMetadata, $matches[1] === 'activity' ? 'diagnostics' : 'stack_trace'); + } + if ($commands) { + foreach ($body['commands'] as $index => $command) { + if (is_array($command) && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) { + $candidate['commands'][$index] = $this->command($command, $messageLimit, $stackLimit, $dropMetadata); + } + } + } + try { + if (JsonDocument::size($candidate) + strlen($path) <= $limit) { + return $candidate; + } + } catch (\JsonException $error) { + throw new FailureReportRefused($limit, $error); + } + if (!$dropMetadata) { + $dropMetadata = true; + } else { + $messageLimit = max(96, intdiv($messageLimit, 2)); + $stackLimit = max(512, intdiv($stackLimit, 2)); + } + } + + throw new FailureReportRefused($limit); + } + + /** @param array $original + * @return array + */ + private function report(array $original, int $messageLimit, int $stackLimit, bool $dropMetadata, string $carrier, array $additionalChanges = []): array + { + $report = $original; + $message = FailureText::bound((string) ($original['message'] ?? ''), $messageLimit, 'message'); + $stack = FailureText::bound((string) ($original['stack_trace'] ?? ''), $stackLimit, 'stack_trace'); + $changes = $additionalChanges; + foreach (['message' => $message, 'stack_trace' => $stack] as $field => $text) { + if ($text['changed']) { + $changes[$field] = ['original_bytes' => $text['original_bytes']]; + } + } + $report['message'] = $message['value']; + if (array_key_exists('stack_trace', $original)) { + $report['stack_trace'] = $stack['value']; + } + $metadata = array_intersect_key($original, array_flip(self::METADATA)); + try { + $metadataBytes = JsonDocument::size($metadata); + } catch (\JsonException) { + $metadataBytes = null; + } + if ($dropMetadata || $metadataBytes === null || $metadataBytes > FailureReportLimits::METADATA_BYTES) { + $this->omitMetadata($report, $metadata, $changes); + } + if ($changes === []) { + return $report; + } + $message = FailureText::bound((string) ($original['message'] ?? ''), $messageLimit, 'message', true); + $report['message'] = $message['value']; + if ($carrier === 'diagnostics') { + $diagnostics = ['truncation' => ['fields' => $changes]]; + if (isset($report['diagnostics'])) { + $diagnostics['original'] = $report['diagnostics']; + } + $report['diagnostics'] = $diagnostics; + if (JsonDocument::size(array_intersect_key($report, array_flip(self::METADATA))) > FailureReportLimits::METADATA_BYTES) { + $this->omitMetadata($report, $metadata, $changes); + $report['diagnostics'] = ['truncation' => ['fields' => $changes]]; + } + } else { + // This endpoint admits a stack string, but no diagnostics field. + $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['fields' => $changes]); + $available = $stackLimit - strlen($suffix); + $stack = FailureText::bound((string) ($original['stack_trace'] ?? ''), $available, 'stack_trace'); + $report['stack_trace'] = $stack['value'].$suffix; + } + + return $report; + } + + /** @param array $report + * @param array $metadata + * @param array $changes + */ + private function omitMetadata(array &$report, array $metadata, array &$changes): void + { + foreach ($metadata as $field => $value) { + try { + $size = JsonDocument::size($value); + } catch (\JsonException) { + $size = null; + } + $changes[$field] = ['omitted' => true, 'original_serialized_bytes' => $size]; + unset($report[$field]); + } + } + + /** @param array $command + * @return array + */ + private function command(array $command, int $messageLimit, int $stackLimit, bool $dropMetadata): array + { + if (($command['type'] ?? null) === 'fail_update') { + $originalMessage = (string) ($command['message'] ?? ''); + $bounded = FailureText::bound($originalMessage, $messageLimit, 'message'); + $command['message'] = $bounded['value']; + if ($bounded['changed']) { + $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['original_message_bytes' => strlen($originalMessage)]); + $command['message'] = FailureText::bound($originalMessage, + $messageLimit - strlen($suffix), 'message', true)['value'].$suffix; + } + + return $command; + } + $exception = is_array($command['exception'] ?? null) ? $command['exception'] : []; + $fixed = array_intersect_key($exception, array_flip(['class', 'type', 'code', 'line'])); + $extra = array_diff_key($exception, $fixed + array_flip(['message', 'stack_trace', 'diagnostics'])); + $report = ['message' => (string) ($command['message'] ?? '')]; + foreach (['stack_trace', 'diagnostics'] as $field) { + if (array_key_exists($field, $exception)) { + $report[$field] = $exception[$field]; + } + } + if ($extra !== []) { + $report['details'] = $extra; + } + $innerMessage = array_key_exists('message', $exception) + ? FailureText::bound((string) $exception['message'], $messageLimit, 'exception.message') : null; + $additionalChanges = $innerMessage !== null && $innerMessage['changed'] + ? ['exception.message' => ['original_bytes' => $innerMessage['original_bytes']]] : []; + $bounded = $this->report($report, $messageLimit, $stackLimit, $dropMetadata, 'diagnostics', $additionalChanges); + $command['message'] = $bounded['message']; + $result = $fixed + ($bounded['details'] ?? []); + if ($innerMessage !== null) { + $result['message'] = $innerMessage['value']; + } + foreach (['stack_trace', 'diagnostics'] as $field) { + if (array_key_exists($field, $bounded)) { + $result[$field] = $bounded[$field]; + } + } + if ($result !== [] || array_key_exists('exception', $command)) { + $command['exception'] = $result; + } + + return $command; + } +} diff --git a/src/Failure/FailureReportLimits.php b/src/Failure/FailureReportLimits.php new file mode 100644 index 00000000..7d44b044 --- /dev/null +++ b/src/Failure/FailureReportLimits.php @@ -0,0 +1,22 @@ + self::REQUEST_BYTES) { + throw new \InvalidArgumentException('Failure-report request bytes must be between 1 and 1048576.'); + } + } +} diff --git a/src/Failure/FailureText.php b/src/Failure/FailureText.php new file mode 100644 index 00000000..1c602253 --- /dev/null +++ b/src/Failure/FailureText.php @@ -0,0 +1,44 @@ + $original, 'changed' => false, 'original_bytes' => $bytes, 'retained_bytes' => $bytes]; + } + $marker = "\n[truncated failure report; original {$field} {$bytes} bytes]"; + $available = $limit - strlen($marker); + if ($available < 0) { + throw new FailureReportRefused($limit); + } + $prefix = substr($original, 0, $available); + $candidate = $prefix; + // A cut through a valid UTF-8 code point needs at most three bytes of retreat. + for ($retreat = 0; $retreat < 3 && preg_match('//u', $candidate) !== 1; ++$retreat) { + $candidate = substr($candidate, 0, -1); + } + if (preg_match('//u', $candidate) !== 1) { + // Repair only the bounded prefix, never allocate an escaped copy of + // an arbitrarily large original just to retain its first64KiB. + $candidate = json_decode(json_encode($prefix, + JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE | JSON_UNESCAPED_UNICODE), true, flags: JSON_THROW_ON_ERROR); + $candidate = substr($candidate, 0, $available); + while (preg_match('//u', $candidate) !== 1) { + $candidate = substr($candidate, 0, -1); + } + } + + return ['value' => $candidate.$marker, 'changed' => true, + 'original_bytes' => $bytes, 'retained_bytes' => strlen($candidate)]; + } +} diff --git a/src/Transport/JsonDocument.php b/src/Transport/JsonDocument.php new file mode 100644 index 00000000..e12bb45d --- /dev/null +++ b/src/Transport/JsonDocument.php @@ -0,0 +1,20 @@ +withHeader($name, $value); } if ($body !== null) { - $json = json_encode($body, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION - | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + $json = JsonDocument::encode($body); $request = $request->withBody($this->streamFactory->createStream($json)); } diff --git a/src/Transport/RequestSizeLimitTransport.php b/src/Transport/RequestSizeLimitTransport.php new file mode 100644 index 00000000..3f33bbe2 --- /dev/null +++ b/src/Transport/RequestSizeLimitTransport.php @@ -0,0 +1,11 @@ +getMessage(), $failure::class, + stackTrace: $failure->getTraceAsString(), ); }, 'activity' => function (Throwable $failure) use ($taskId, $leaseOwner, $task): void { @@ -1303,6 +1304,7 @@ private function rejectPolledTask(string $taskKind, array $task, CodecException $failure->getMessage(), $failure::class, nonRetryable: true, + stackTrace: $failure->getTraceAsString(), ); }, 'query' => function (Throwable $failure) use ($taskId, $leaseOwner, $task): void { @@ -1435,6 +1437,7 @@ function (Throwable $failure) use ($taskId, $leaseOwner, $attempt): void { $failure::class, $failure instanceof NonDeterministicWorkflow || $failure instanceof WorkflowEntryArgumentsMismatch ? $failure->reason : null, $failure instanceof NonDeterministicWorkflow ? $failure->sequence : null, + $failure->getTraceAsString(), ); }, $attempt, @@ -2536,6 +2539,7 @@ function (Throwable $failure) use ($taskId, $attemptId, $leaseOwner): void { $failure->getMessage(), $failure instanceof ActivityExecutionFailure ? $failure->originalType : $failure::class, $failure instanceof ActivityCancelled || ($failure instanceof ActivityExecutionFailure && $failure->cancelled), + stackTrace: $failure->getTraceAsString(), ); }, ); @@ -3945,22 +3949,13 @@ private function workflowFailureCommand( 'type' => 'fail_workflow', 'message' => $exception->getMessage(), 'exception_type' => $exception::class, + 'exception' => ['stack_trace' => $exception->getTraceAsString()], ]; if ($failedActivitySequence !== null && $failedActivityExecutionId !== null) { $command['failed_step_sequence'] = $failedActivitySequence; $command['failed_activity_execution_id'] = $failedActivityExecutionId; } - try { - json_encode($command, JSON_THROW_ON_ERROR); - } catch (\JsonException) { - $command = [ - 'type' => 'fail_workflow', - 'message' => 'Workflow failure metadata could not be encoded for the HTTP JSON wire boundary.', - 'exception_type' => \RuntimeException::class, - ]; - } - return $command; } diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php new file mode 100644 index 00000000..db3a1198 --- /dev/null +++ b/tests/FailureReportTest.php @@ -0,0 +1,192 @@ +requests[] = $request; + return new Response(200, ['Content-Type' => 'application/json'], '{}'); + } + }; + $client = new Client('https://recording.invalid', transport: new Psr18Transport(client: $http)); + $large = str_repeat('m', 3 * 1024 * 1024); + $message = in_array($scenario, ['message', 'combined'], true) ? $large : 'original message'; + $stack = in_array($scenario, ['stack', 'combined'], true) ? $large : null; + if ($scenario === 'escaped') { + $message = str_repeat("\0\"\\雪🦀", 300000); + $stack = str_repeat("\0\"\\雪🦀", 300000); + } + $details = $scenario === 'details' ? ['nested' => ['cause' => $large]] : ['whole' => 1.0, 'zero' => -0.0]; + $client->failActivityTask('original-task', 'original-attempt', 'original-owner', $message, + 'OriginalFailure', true, $details, $stack, ['authored_retry_delay' => 12.5]); + $reports = array_values(array_filter($http->requests, + static fn (RequestInterface $request): bool => str_ends_with($request->getUri()->getPath(), '/fail'))); + self::assertCount(1, $reports); + $request = $reports[0]; + $wire = (string) $request->getBody(); + self::assertLessThanOrEqual(1048576, strlen($wire)); + self::assertSame('/api/worker/activity-tasks/original-task/fail', $request->getUri()->getPath()); + $body = json_decode($wire, true, flags: JSON_THROW_ON_ERROR); + self::assertSame('original-attempt', $body['activity_attempt_id']); + self::assertSame('original-owner', $body['lease_owner']); + $failure = $body['failure']; + self::assertSame('OriginalFailure', $failure['type']); + self::assertTrue($failure['non_retryable']); + self::assertLessThanOrEqual(65536, strlen($failure['message'])); + self::assertLessThanOrEqual(131072, strlen($failure['stack_trace'] ?? '')); + self::assertSame(1, preg_match('//u', $failure['message'])); + $metadata = array_intersect_key($failure, array_flip(['details', 'diagnostics', 'runtime_diagnostics'])); + self::assertLessThanOrEqual(65536, JsonDocument::size($metadata)); + if ($scenario === 'small') { + self::assertSame($message, $failure['message']); + self::assertSame(['authored_retry_delay' => 12.5], $failure['diagnostics']); + } else { + self::assertStringContainsString('[truncated failure report;', $failure['message']); + self::assertNotEmpty($failure['diagnostics']['truncation']['fields']); + } + if ($scenario === 'details') { + self::assertArrayNotHasKey('details', $failure); + self::assertTrue($failure['diagnostics']['truncation']['fields']['details']['omitted']); + } else { + self::assertSame($client->payloadCodec()->envelope($details), $failure['details']); + self::assertSame($details, $client->payloadCodec()->decodeEnvelope($failure['details'])); + } + } + + public static function largeReports(): iterable + { + foreach (['message', 'stack', 'combined', 'escaped', 'details', 'small'] as $case) { + yield $case => [$case]; + } + } + + public function testWorkflowTaskUsesOnlyPublishedStackCarrier(): void + { + $transport = new FakeTransport([[]]); + $client = new Client('http://recording.invalid', transport: $transport); + $client->failWorkflowTask('task', 'owner', 7, str_repeat('m', 3 * 1024 * 1024), + 'OriginalFailure', 'original-reason', 9, str_repeat('s', 3 * 1024 * 1024)); + $body = $transport->requests[0]['body']; + self::assertSame('owner', $body['lease_owner']); + self::assertSame(7, $body['workflow_task_attempt']); + $failure = $body['failure']; + self::assertSame('OriginalFailure', $failure['type']); + self::assertSame('original-reason', $failure['reason']); + self::assertSame(9, $failure['sequence']); + self::assertArrayNotHasKey('diagnostics', $failure); + self::assertStringContainsString('[truncation diagnostics]', $failure['stack_trace']); + self::assertStringContainsString('3145728', $failure['stack_trace']); + self::assertLessThanOrEqual(131072, strlen($failure['stack_trace'])); + self::assertLessThanOrEqual(1048576, JsonDocument::size($body)); + } + + public function testCompletionBoundsNestedExceptionWithoutChangingClassification(): void + { + $transport = new FakeTransport([[]]); + $client = new Client('http://recording.invalid', transport: $transport); + $large = str_repeat('x', 3 * 1024 * 1024); + $command = ['type' => 'fail_workflow', 'message' => 'outer', 'exception_type' => 'OriginalFailure', + 'non_retryable' => false, 'failed_step_sequence' => 8, 'failed_activity_execution_id' => 'activity', + 'exception' => ['message' => $large, 'properties' => ['previous' => ['message' => $large]], 'stack_trace' => $large]]; + $client->completeWorkflowTask('task', 'owner', 4, [$command]); + $body = $transport->requests[0]['body']; + $bounded = $body['commands'][0]; + foreach (['type', 'exception_type', 'non_retryable', 'failed_step_sequence', 'failed_activity_execution_id'] as $key) { + self::assertSame($command[$key], $bounded[$key]); + } + self::assertArrayNotHasKey('class', $bounded['exception']); + self::assertArrayNotHasKey('type', $bounded['exception']); + self::assertArrayNotHasKey('properties', $bounded['exception']); + self::assertStringContainsString('[truncated failure report;', $bounded['message']); + self::assertStringContainsString('[truncated failure report;', $bounded['exception']['message']); + self::assertTrue($bounded['exception']['diagnostics']['truncation']['fields']['details']['omitted']); + self::assertSame('owner', $body['lease_owner']); + self::assertSame(4, $body['workflow_task_attempt']); + } + + public function testLowerConfiguredLimitCountsEscapingAfterAllMarkers(): void + { + $transport = new FakeTransport([[]]); + $client = (new Client('http://recording.invalid', transport: $transport))->withFailureReportLimit(4096); + $client->failActivityTask('task', 'attempt', 'owner', str_repeat("\0\"\\雪", 600000), 'OriginalFailure', false, + ['oversized' => str_repeat('d', 100000)], str_repeat("\0", 3000000)); + $body = $transport->requests[0]['body']; + self::assertLessThanOrEqual(4096, JsonDocument::size($body)); + self::assertFalse($body['failure']['non_retryable']); + self::assertStringContainsString('[truncated failure report;', $body['failure']['message']); + self::assertArrayNotHasKey('details', $body['failure']); + } + + public function testKnownTransportLimitIsRespectedAndImpossibleIdentityRefusesBeforeSend(): void + { + $transport = new class implements Transport, RequestSizeLimitTransport { + public array $requests = []; + public function maximumRequestBytes(): ?int { return 4096; } + public function send(string $method, string $uri, array $headers, ?array $body = null): ?array + { + $this->requests[] = $body; + return []; + } + }; + $client = new Client('http://recording.invalid', transport: $transport); + $client->failActivityTask('task', 'attempt', 'owner', str_repeat('x', 3000000)); + self::assertLessThanOrEqual(4096, JsonDocument::size($transport->requests[0])); + try { + $client->failActivityTask('task', str_repeat('identity', 1000), 'owner', 'small', 'OriginalFailure'); + self::fail('The fixed identity cannot fit.'); + } catch (FailureReportRefused $error) { + self::assertSame('failure_report_refused', $error->response['reason']); + self::assertFalse($error->transientConnectionFailure); + } + self::assertCount(1, $transport->requests); + } + + public function testAggregateCompletionBudgetIncludesEveryCommandAndFixedBusinessContent(): void + { + $limiter = new FailureReportLimiter(new FailureReportLimits(), new FakeTransport()); + $command = ['type' => 'fail_workflow', 'message' => str_repeat("\0", 3000000), 'exception_type' => 'Original']; + $body = ['lease_owner' => 'owner', 'workflow_task_attempt' => 3, 'commands' => array_fill(0, 8, $command)]; + $result = $limiter->request('/worker/workflow-tasks/task/complete', $body); + self::assertLessThanOrEqual(1048576, JsonDocument::size($result)); + self::assertCount(8, $result['commands']); + $body['commands'][] = ['type' => 'record_marker', 'name' => 'business', 'details' => str_repeat('x', 1100000)]; + $this->expectException(FailureReportRefused::class); + $limiter->request('/worker/workflow-tasks/task/complete', $body); + } + + public function testInvalidUtf8MessageRepairsBoundedPrefixWithoutChangingType(): void + { + $transport = new FakeTransport([[]]); + $client = new Client('http://recording.invalid', transport: $transport); + $client->failActivityTask('task', 'attempt', 'owner', "broken\xff message", 'OriginalType'); + $failure = $transport->requests[0]['body']['failure']; + self::assertSame('OriginalType', $failure['type']); + self::assertSame(1, preg_match('//u', $failure['message'])); + self::assertStringContainsString('[truncated failure report;', $failure['message']); + } +} From 09f4691e1a13597937d8f11421c3b49ce4753351 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:09:37 +0000 Subject: [PATCH 02/19] Qualify published failure delivery and refuse unencodable identities --- scripts/qualify-failure-report-delivery.php | 152 ++++++++++++++++++++ src/Client.php | 1 + src/Failure/FailureReportLimiter.php | 1 + src/Worker.php | 4 + tests/PortableWorkerAffinityTest.php | 49 +------ tests/Support/control-timeout-peer.php | 3 +- 6 files changed, 167 insertions(+), 43 deletions(-) create mode 100644 scripts/qualify-failure-report-delivery.php diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php new file mode 100644 index 00000000..ecbd6f9b --- /dev/null +++ b/scripts/qualify-failure-report-delivery.php @@ -0,0 +1,152 @@ +client = new GuzzleHttp\Client(['http_errors' => false, 'timeout' => 15]); } + public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Http\Message\ResponseInterface + { + $wire = (string) $request->getBody(); + $data = $wire === '' ? [] : json_decode($wire, true, flags: JSON_THROW_ON_ERROR); + $response = $this->client->sendRequest($request); + if ($response->getStatusCode() >= 400) { + fwrite(STDERR, json_encode(['path' => $request->getUri()->getPath(), 'status' => $response->getStatusCode(), + 'response' => array_diff_key(json_decode((string) $response->getBody(), true), ['server_capabilities' => true])], JSON_THROW_ON_ERROR)."\n"); + } + $path = $request->getUri()->getPath(); + if (str_ends_with($path, '/fail') || isset($data['commands'][0]['exception_type'])) { + $failure = $data['failure'] ?? $data['commands'][0]; + $this->reports[] = ['path' => $path, 'encoded_bytes' => strlen($wire), 'http_status' => $response->getStatusCode(), + 'lease_owner' => $data['lease_owner'] ?? null, 'activity_attempt_id' => $data['activity_attempt_id'] ?? null, + 'workflow_task_attempt' => $data['workflow_task_attempt'] ?? null, + 'type' => $failure['type'] ?? $failure['exception_type'] ?? null, + 'message_bytes' => strlen($failure['message']), + 'marked' => str_contains($failure['message'], '[truncated failure report;'), + 'stack_bytes' => strlen($failure['stack_trace'] ?? $failure['exception']['stack_trace'] ?? ''), + 'message_sha256' => hash('sha256', $failure['message']), 'non_retryable' => $failure['non_retryable'] ?? null]; + } + return $response; + } +}; +$namespace = 'php-failure-'.bin2hex(random_bytes(6)); +$admin = new Client($url, controlToken: 'claimed-task-admin'); +$admin->createNamespace($namespace); +$control = new Client($url, namespace: $namespace, controlToken: 'claimed-task-client'); +$transport = new DurableWorkflow\Transport\Psr18Transport(client: $http); +$client = new Client($url, namespace: $namespace, transport: $transport, workerToken: 'claimed-task-worker'); +$queue = $namespace; +$workerId = $namespace.'-workflow'; +$activityWorkerId = $namespace.'-activity'; +$worker = new Worker($client, $queue, workerId: $workerId, stickyCacheCapacity: 1); +$worker->registerWorkflow('qualification.failure', static function (WorkflowContext $context, string $kind): array { + if ($kind === 'terminal') { + throw new FailureDeliveryQualificationException(str_repeat('t', 3 * 1024 * 1024)); + } + if ($kind === 'workflow-task') { + return ['fresh_turn' => true]; + } + try { + $context->activity('qualification.failure.activity', [], ['retry_policy' => ['max_attempts' => 1]]); + throw new FailureDeliveryQualificationException('The activity unexpectedly succeeded.'); + } catch (ActivityFailed $failure) { + return ['type' => $failure->failureType, 'marked' => str_contains($failure->getMessage(), '[truncated failure report;'), + 'message_bytes' => strlen($failure->getMessage()), 'non_retryable' => $failure->nonRetryable]; + } +}); +$client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes'], + workflowCommandContracts: $worker->contracts()['workflow_commands']); +$client->registerWorker($activityWorkerId, $queue, [], ['qualification.failure.activity']); +$result = ['sdk_version' => Composer\InstalledVersions::getPrettyVersion('durable-workflow/sdk'), + 'sdk_reference' => Composer\InstalledVersions::getReference('durable-workflow/sdk'), + 'server_version' => $admin->clusterInfo()->version, 'namespace' => $namespace, 'scenarios' => []]; +register_shutdown_function(static function () use (&$result, $http): void { + if (!isset($result['pass'])) { + $result['reports'] = $http->reports; + $result['incomplete'] = true; + $result['last_error'] = error_get_last(); + echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; + } +}); +$large = str_repeat('m', 3 * 1024 * 1024); +foreach (['message', 'stack', 'combined', 'escaped', 'details', 'small'] as $kind) { + $handle = $control->startWorkflow('qualification.failure', $namespace.'-'.$kind, $queue, [$kind], runTimeoutSeconds: 120); + $worker->tick(0); + $task = $client->pollActivityTask($activityWorkerId, $queue, 0); + requireDelivery(is_array($task), 'The activity was not leased for '.$kind); + $message = in_array($kind, ['message', 'combined'], true) ? $large : 'original message'; + $stack = in_array($kind, ['stack', 'combined'], true) ? $large : null; + if ($kind === 'escaped') { + $message = $stack = str_repeat("\0\"\\雪🦀", 300000); + } + $details = $kind === 'details' ? ['nested' => ['cause' => $large]] : ['whole' => 1.0, 'zero' => -0.0]; + $type = 'OriginalFailure.'.$kind; + $client->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], + $message, $type, true, $details, $stack); + $worker->tick(0); + $output = $handle->resultOfSelectedRun(15, 0.05); + requireDelivery($output['type'] === $type && $output['marked'] === ($kind !== 'small'), 'Original workflow failure mismatch.'); + requireDelivery($output['non_retryable'] === true, 'The non-retryable decision changed.'); + $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); + $events = $history['events'] ?? $history['history_events'] ?? []; + $counts = array_count_values(array_column($events, 'event_type')); + requireDelivery(($counts['ActivityFailed'] ?? 0) === 1 && ($counts['WorkflowCompleted'] ?? 0) === 1, 'Failure history missing.'); + $result['scenarios'][$kind] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'task_id' => $task['task_id'], 'attempt_id' => $task['activity_attempt_id'] ?? $task['attempt_id'], + 'lease_owner' => $task['lease_owner'], 'output' => $output, 'event_counts' => $counts]; +} +$handle = $control->startWorkflow('qualification.failure', $namespace.'-workflow-task', $queue, ['workflow-task'], runTimeoutSeconds: 120); +$task = $client->pollWorkflowTask($workerId, $queue, 0); +requireDelivery(is_array($task), 'Original workflow task not leased.'); +$ack = $client->failWorkflowTask($task['task_id'], $task['lease_owner'], $task['workflow_task_attempt'], $large, + 'OriginalWorkflowTaskFailure', stackTrace: $large); +requireDelivery(($ack['recorded'] ?? null) === true, 'Workflow-task failure not recorded.'); +$worker->tick(0); +$output = $handle->resultOfSelectedRun(15, 0.05); +requireDelivery($output === ['fresh_turn' => true], 'The workflow did not receive its fresh legitimate turn.'); +$history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); +$events = $history['events'] ?? $history['history_events'] ?? []; + +$failureEvents = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'WorkflowTaskFailed')); +// The published runtime stores task failure separately and emits no history event. +requireDelivery(($ack['next_task_id'] ?? null) !== $task['task_id'], 'Failed claim identity was reused as retry authority.'); +$result['scenarios']['workflow-task'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'original_task_id' => $task['task_id'], 'lease_owner' => $task['lease_owner'], 'attempt' => $task['workflow_task_attempt'], + 'ack' => $ack, 'history_failure_events' => count($failureEvents), 'fresh_turn_result' => $output, + 'history_visibility' => 'Published runtime emits no WorkflowTaskFailed history event; parent225 runtime follow-through remains.']; +$handle = $control->startWorkflow('qualification.failure', $namespace.'-terminal', $queue, ['terminal'], runTimeoutSeconds: 120); +$worker->tick(0); +try { + $handle->resultOfSelectedRun(15, 0.05); + throw new FailureDeliveryQualificationException('Terminal workflow unexpectedly succeeded.'); +} catch (WorkflowFailed $failure) { + requireDelivery($failure->failureType === FailureDeliveryQualificationException::class, 'Workflow failure type changed.'); + requireDelivery(str_contains($failure->getMessage(), '[truncated failure report;'), 'Terminal workflow marker missing.'); + $result['scenarios']['terminal'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'failure_type' => $failure->failureType, 'marked_message' => true, 'message_bytes' => strlen($failure->getMessage())]; +} +foreach ($http->reports as $report) { + requireDelivery($report['encoded_bytes'] <= 1048576 && $report['http_status'] === 200, 'Encoded/reporting HTTP boundary failed.'); +} +$result['reports'] = $http->reports; +$result['pass'] = true; +echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; diff --git a/src/Client.php b/src/Client.php index b3f9e0b7..b12a0552 100644 --- a/src/Client.php +++ b/src/Client.php @@ -2213,6 +2213,7 @@ public function completeActivityTask( /** * @param array|null $details + * @param array|null $diagnostics * @return array */ public function failActivityTask( diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index e30ecff8..acd332c2 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -81,6 +81,7 @@ public function request(string $path, array $body): array } /** @param array $original + * @param array $additionalChanges * @return array */ private function report(array $original, int $messageLimit, int $stackLimit, bool $dropMetadata, string $carrier, array $additionalChanges = []): array diff --git a/src/Worker.php b/src/Worker.php index 02939cf4..e2bc3231 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -6,6 +6,7 @@ use DurableWorkflow\Exception\ActivityCancelled; use DurableWorkflow\Exception\CodecException; +use DurableWorkflow\Exception\FailureReportRefused; use DurableWorkflow\Exception\InvalidLocalActivityReport; use DurableWorkflow\Exception\InvalidWorkerDefinition; use DurableWorkflow\Exception\LocalActivityTimedOut; @@ -2802,6 +2803,9 @@ private function acknowledgeTaskFailure( callable $failureAcknowledgement, ?int $workflowTaskAttempt = null, ): void { + if ($taskFailure instanceof FailureReportRefused) { + throw $taskFailure; + } if ($this->isTaskAuthorityLost($taskKind, $taskId, $taskFailure, $workflowTaskAttempt)) { if ($taskKind === 'workflow' && $taskFailure instanceof ServerException && ($taskFailure->details['task_status'] ?? null) === 'ready') { diff --git a/tests/PortableWorkerAffinityTest.php b/tests/PortableWorkerAffinityTest.php index 10bd391c..5f52e3a0 100644 --- a/tests/PortableWorkerAffinityTest.php +++ b/tests/PortableWorkerAffinityTest.php @@ -1530,51 +1530,16 @@ public function testCompletedLocalActivityIsRecordedWhenWorkflowThrowsAndColdRep self::assertSame(\RuntimeException::class, $replayed[0]['exception_type'] ?? null); } - public function testJsonUnsafeWorkflowFailureIsNormalizedOnInitialExecutionAndColdReplay(): void + public function testJsonUnsafeWorkflowFailureTypeExplicitlyRefusesBeforeChangingIdentity(): void { - $outcome = 'completed_then_json_unsafe_workflow_failed'; $calls = 0; $now = 0.0; - $initial = $this->runTerminalLocalActivityWorkflow($outcome, [], $calls, $now); - - self::assertCount(2, $initial); - self::assertSame('record_local_activity', $initial[0]['type'] ?? null); - self::assertSame('completed', $initial[0]['outcome'] ?? null); - self::assertSame('fail_workflow', $initial[1]['type'] ?? null); - self::assertSame( - 'Workflow failure metadata could not be encoded for the HTTP JSON wire boundary.', - $initial[1]['message'] ?? null, - ); - self::assertSame(\RuntimeException::class, $initial[1]['exception_type'] ?? null); - self::assertJson(json_encode($initial, JSON_THROW_ON_ERROR)); - self::assertSame(1, $calls); - - $history = [ - [ - 'event_type' => 'ActivityScheduled', - 'payload' => [ - 'sequence' => 1, - 'activity_type' => 'terminal-local-activity', - 'execution_mode' => 'local', - ], - ], - [ - 'event_type' => 'ActivityCompleted', - 'payload' => [ - 'sequence' => 1, - 'activity_type' => 'terminal-local-activity', - 'execution_mode' => 'local', - 'outcome' => 'completed', - 'result' => $initial[0]['result'] ?? null, - 'attempts' => $initial[0]['attempts'] ?? [], - ], - ], - ]; - $replayed = $this->runTerminalLocalActivityWorkflow($outcome, $history, $calls, $now); - - self::assertSame(1, $calls, 'Cold replay repeated the completed local activity side effect.'); - self::assertSame([$initial[1]], $replayed); - self::assertJson(json_encode($replayed, JSON_THROW_ON_ERROR)); + $this->expectException(\DurableWorkflow\Exception\FailureReportRefused::class); + try { + $this->runTerminalLocalActivityWorkflow('completed_then_json_unsafe_workflow_failed', [], $calls, $now); + } finally { + self::assertSame(1, $calls); + } } public function testTypedWorkerSessionLifecycleIsIdempotentAndSupportsReacquisition(): void diff --git a/tests/Support/control-timeout-peer.php b/tests/Support/control-timeout-peer.php index d425c5e4..8a01c5db 100644 --- a/tests/Support/control-timeout-peer.php +++ b/tests/Support/control-timeout-peer.php @@ -10,7 +10,8 @@ if ($server === false) { exit(1); } -file_put_contents($directory.'/address', stream_socket_get_name($server, false)); +file_put_contents($directory.'/address.tmp', stream_socket_get_name($server, false)); +rename($directory.'/address.tmp', $directory.'/address'); $until = microtime(true) + 30; while (microtime(true) < $until) { $peer = @stream_socket_accept($server, 1); From ed0b2fe6d3efd02f7b7e2e03be44753535b840ec Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:19:13 +0000 Subject: [PATCH 03/19] Bound cooperative failure IPC independently of payload transfers --- docs/portal/build/failures-retries.md | 41 ++++++++++++ scripts/qualify-failure-report-delivery.php | 44 +++++++++++-- src/Failure/FailureReportLimiter.php | 72 ++++++++++++++++---- src/Failure/FailureText.php | 15 ++++- src/Worker.php | 3 +- src/Worker/ActivityExecutionFailure.php | 3 + src/Worker/CooperativeActivityExecutor.php | 73 ++++++++++++++------- tests/CooperativeActivityExecutorTest.php | 28 ++++++++ tests/FailureReportTest.php | 17 +++++ 9 files changed, 252 insertions(+), 44 deletions(-) diff --git a/docs/portal/build/failures-retries.md b/docs/portal/build/failures-retries.md index e6801a7e..7c993e06 100644 --- a/docs/portal/build/failures-retries.md +++ b/docs/portal/build/failures-retries.md @@ -107,3 +107,44 @@ A client-side wait timeout is not proof that the workflow failed; describe the h Do not paper over `NonDeterministicWorkflow` with process retries. Compare the deployed build with the history's worker routing, restore the old command sequence, and roll forward with a replay-compatible change. Keep a replay corpus for long-lived workflow histories in [testing](/build/testing/).
Retry question

Before adding a retry, identify the durable request ID, the side effect's idempotency boundary, and what evidence proves the prior attempt did not already succeed.

+ +## Bound failure diagnostics + +Failure reporting keeps diagnostic content within finite defaults: + +| Content | Default budget | +| --- | --- | +| UTF-8 message, including its marker | 64 KiB | +| Stack trace, including diagnostic markers | 128 KiB | +| Serialized details and diagnostics together | 64 KiB | +| Complete encoded failure request, including JSON escaping and identity | 1 MiB | + +Oversized content produces a visible `[truncated failure report; ...]` message +marker and bounded diagnostics with original sizes and omitted fields. The SDK +retains a complete valid Avro details envelope or explicitly omits it. It never +slices Avro or base64 bytes. Small reports keep their message and details. + +The original failure type, retry decision, authored retry delays and claimed +attempt identity remain unchanged. Diagnostic limits are separate from business +payload and worker request timeout budgets. Cooperative callbacks also bound +failure diagnostics before sending them to their owning worker. + +For a transport with a smaller request budget, narrow the client limit: + +```php +$client = $client->withFailureReportLimit(128 * 1024); +``` + +An injected transport can expose a lower known body budget through +`RequestSizeLimitTransport`. The lower limit applies after markers and JSON +escaping are included. If required identity or fixed request content cannot fit, +`FailureReportRefused` reports `failure_report_refused` locally. Treat this as a +reporting refusal requiring configuration or identity repair, rather than a new +application failure or permission to publish under a different attempt. + +Activity failures use existing diagnostic fields. Workflow-task failures use +the existing stack string for structured truncation information. The published +Server qualification records task failures and grants a legitimate retry turn, +but that runtime does not append a workflow-task failure event to workflow +history or persist its stack diagnostics. That visibility work remains tracked +in [shared issue 225](https://github.com/durable-workflow/.github/issues/225). diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index ecbd6f9b..99de7498 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -39,7 +39,7 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt $this->reports[] = ['path' => $path, 'encoded_bytes' => strlen($wire), 'http_status' => $response->getStatusCode(), 'lease_owner' => $data['lease_owner'] ?? null, 'activity_attempt_id' => $data['activity_attempt_id'] ?? null, 'workflow_task_attempt' => $data['workflow_task_attempt'] ?? null, - 'type' => $failure['type'] ?? $failure['exception_type'] ?? null, + 'type' => isset($data['failure']) ? ($failure['type'] ?? null) : ($failure['exception_type'] ?? null), 'message_bytes' => strlen($failure['message']), 'marked' => str_contains($failure['message'], '[truncated failure report;'), 'stack_bytes' => strlen($failure['stack_trace'] ?? $failure['exception']['stack_trace'] ?? ''), @@ -62,6 +62,10 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt if ($kind === 'terminal') { throw new FailureDeliveryQualificationException(str_repeat('t', 3 * 1024 * 1024)); } + if ($kind === 'retry') { + return ['activity_value' => $context->activity('qualification.failure.activity', [], + ['retry_policy' => ['max_attempts' => 2, 'backoff_seconds' => [1]]])]; + } if ($kind === 'workflow-task') { return ['fresh_turn' => true]; } @@ -88,19 +92,20 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt } }); $large = str_repeat('m', 3 * 1024 * 1024); -foreach (['message', 'stack', 'combined', 'escaped', 'details', 'small'] as $kind) { +foreach (['message', 'stack', 'combined', 'escaped', 'details', 'lower-limit', 'small'] as $kind) { $handle = $control->startWorkflow('qualification.failure', $namespace.'-'.$kind, $queue, [$kind], runTimeoutSeconds: 120); $worker->tick(0); $task = $client->pollActivityTask($activityWorkerId, $queue, 0); requireDelivery(is_array($task), 'The activity was not leased for '.$kind); - $message = in_array($kind, ['message', 'combined'], true) ? $large : 'original message'; - $stack = in_array($kind, ['stack', 'combined'], true) ? $large : null; + $message = in_array($kind, ['message', 'combined', 'lower-limit'], true) ? $large : 'original message'; + $stack = in_array($kind, ['stack', 'combined', 'lower-limit'], true) ? $large : null; if ($kind === 'escaped') { $message = $stack = str_repeat("\0\"\\雪🦀", 300000); } $details = $kind === 'details' ? ['nested' => ['cause' => $large]] : ['whole' => 1.0, 'zero' => -0.0]; $type = 'OriginalFailure.'.$kind; - $client->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], + $reporter = $kind === 'lower-limit' ? $client->withFailureReportLimit(4096) : $client; + $reporter->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], $message, $type, true, $details, $stack); $worker->tick(0); $output = $handle->resultOfSelectedRun(15, 0.05); @@ -114,6 +119,35 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt 'task_id' => $task['task_id'], 'attempt_id' => $task['activity_attempt_id'] ?? $task['attempt_id'], 'lease_owner' => $task['lease_owner'], 'output' => $output, 'event_counts' => $counts]; } +$handle = $control->startWorkflow('qualification.failure', $namespace.'-retry', $queue, ['retry'], runTimeoutSeconds: 120); +$worker->tick(0); +$original = $client->pollActivityTask($activityWorkerId, $queue, 0); +requireDelivery(is_array($original), 'Retry control original attempt was not leased.'); +$retryStarted = hrtime(true); +$client->failActivityTask($original['task_id'], $original['activity_attempt_id'] ?? $original['attempt_id'], + $original['lease_owner'], $large, 'OriginalRetryableFailure', false); +$deadline = microtime(true) + 15; +do { + $retry = $client->pollActivityTask($activityWorkerId, $queue, 0); + if ($retry !== null) { break; } + usleep(50000); +} while (microtime(true) < $deadline); +requireDelivery(is_array($retry), 'Authored retry was not made available.'); +$originalAttempt = $original['activity_attempt_id'] ?? $original['attempt_id']; +$retryAttempt = $retry['activity_attempt_id'] ?? $retry['attempt_id']; +requireDelivery($retryAttempt !== $originalAttempt, 'The failed attempt was reused as publication authority.'); +$client->completeActivityTask($retry['task_id'], $retryAttempt, $retry['lease_owner'], 42); +$worker->tick(0); +$output = $handle->resultOfSelectedRun(15, 0.05); +requireDelivery($output === ['activity_value' => 42], 'The original workflow did not receive the legitimate retry result.'); +$history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); +$events = $history['events'] ?? $history['history_events'] ?? []; +$scheduled = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityScheduled')); +$result['scenarios']['retry'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'original_task_id' => $original['task_id'], 'original_attempt_id' => $originalAttempt, + 'retry_task_id' => $retry['task_id'], 'retry_attempt_id' => $retryAttempt, + 'retry_elapsed_seconds' => (hrtime(true) - $retryStarted) / 1e9, + 'scheduled_payload' => $scheduled[0]['payload'] ?? null, 'output' => $output]; $handle = $control->startWorkflow('qualification.failure', $namespace.'-workflow-task', $queue, ['workflow-task'], runTimeoutSeconds: 120); $task = $client->pollWorkflowTask($workerId, $queue, 0); requireDelivery(is_array($task), 'Original workflow task not leased.'); diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index acd332c2..1fba830e 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -14,7 +14,7 @@ final class FailureReportLimiter { private const METADATA = ['details', 'diagnostics', 'runtime_diagnostics', 'validation_errors']; - public function __construct(private readonly FailureReportLimits $limits, private readonly Transport $transport) + public function __construct(private readonly FailureReportLimits $limits, private readonly ?Transport $transport = null) { } @@ -46,14 +46,32 @@ public function request(string $path, array $body): array $limit = min($limit, $known); } } + return $this->bound($body, $endpoint ? ($matches[1] === 'activity' ? 'diagnostics' : 'stack_trace') : null, + $commands, $limit, strlen($path)); + } + + /** @internal The callback IPC carries the same bounded report before HTTP admission. + * @param array $body + * @return array + */ + public function activityEnvelope(array $body): array + { + return $this->bound($body, 'diagnostics', false, $this->limits->requestBytes, 0); + } + + /** @param array $body + * @return array + */ + private function bound(array $body, ?string $carrier, bool $commands, int $limit, int $pathBytes): array + { $messageLimit = FailureReportLimits::MESSAGE_BYTES; $stackLimit = FailureReportLimits::STACK_BYTES; $dropMetadata = false; for ($pass = 0; $pass < 16; ++$pass) { $candidate = $body; - if ($endpoint) { + if ($carrier !== null) { $candidate['failure'] = $this->report($body['failure'], $messageLimit, $stackLimit, - $dropMetadata, $matches[1] === 'activity' ? 'diagnostics' : 'stack_trace'); + $dropMetadata, $carrier); } if ($commands) { foreach ($body['commands'] as $index => $command) { @@ -63,7 +81,7 @@ public function request(string $path, array $body): array } } try { - if (JsonDocument::size($candidate) + strlen($path) <= $limit) { + if (JsonDocument::size($candidate) + $pathBytes <= $limit) { return $candidate; } } catch (\JsonException $error) { @@ -72,8 +90,8 @@ public function request(string $path, array $body): array if (!$dropMetadata) { $dropMetadata = true; } else { - $messageLimit = max(96, intdiv($messageLimit, 2)); - $stackLimit = max(512, intdiv($stackLimit, 2)); + $messageLimit = intdiv($messageLimit, 2); + $stackLimit = intdiv($stackLimit, 2); } } @@ -87,8 +105,8 @@ public function request(string $path, array $body): array private function report(array $original, int $messageLimit, int $stackLimit, bool $dropMetadata, string $carrier, array $additionalChanges = []): array { $report = $original; - $message = FailureText::bound((string) ($original['message'] ?? ''), $messageLimit, 'message'); - $stack = FailureText::bound((string) ($original['stack_trace'] ?? ''), $stackLimit, 'stack_trace'); + $message = $this->text((string) ($original['message'] ?? ''), $messageLimit, 'message'); + $stack = $this->text((string) ($original['stack_trace'] ?? ''), $stackLimit, 'stack_trace'); $changes = $additionalChanges; foreach (['message' => $message, 'stack_trace' => $stack] as $field => $text) { if ($text['changed']) { @@ -111,7 +129,8 @@ private function report(array $original, int $messageLimit, int $stackLimit, boo if ($changes === []) { return $report; } - $message = FailureText::bound((string) ($original['message'] ?? ''), $messageLimit, 'message', true); + $changes = $this->priorChanges($original) + $changes; + $message = $this->text((string) ($original['message'] ?? ''), $messageLimit, 'message', true); $report['message'] = $message['value']; if ($carrier === 'diagnostics') { $diagnostics = ['truncation' => ['fields' => $changes]]; @@ -127,13 +146,40 @@ private function report(array $original, int $messageLimit, int $stackLimit, boo // This endpoint admits a stack string, but no diagnostics field. $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['fields' => $changes]); $available = $stackLimit - strlen($suffix); - $stack = FailureText::bound((string) ($original['stack_trace'] ?? ''), $available, 'stack_trace'); + $stack = $this->text((string) ($original['stack_trace'] ?? ''), $available, 'stack_trace'); $report['stack_trace'] = $stack['value'].$suffix; } return $report; } + /** @return array{value: string, changed: bool, original_bytes: int, retained_bytes: int} */ + private function text(string $original, int $limit, string $field, bool $forceMarker = false): array + { + return FailureText::bound($original, max($limit, FailureText::minimumBytes($original, $field)), $field, $forceMarker); + } + + /** Preserve bounded original-size provenance when an IPC report is narrowed again. + * @param array $original + * @return array + */ + private function priorChanges(array $original): array + { + $fields = $original['diagnostics']['truncation']['fields'] ?? []; + if (!is_array($fields)) { + return []; + } + $result = []; + foreach (['message', 'stack_trace', 'exception.message', ...self::METADATA] as $field) { + $entry = $fields[$field] ?? null; + $bytes = is_array($entry) ? ($entry['original_bytes'] ?? null) : null; + if (is_int($bytes) && $bytes >= 0) { + $result[$field] = ['original_bytes' => $bytes]; + } + } + return $result; + } + /** @param array $report * @param array $metadata * @param array $changes @@ -158,11 +204,11 @@ private function command(array $command, int $messageLimit, int $stackLimit, boo { if (($command['type'] ?? null) === 'fail_update') { $originalMessage = (string) ($command['message'] ?? ''); - $bounded = FailureText::bound($originalMessage, $messageLimit, 'message'); + $bounded = $this->text($originalMessage, $messageLimit, 'message'); $command['message'] = $bounded['value']; if ($bounded['changed']) { $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['original_message_bytes' => strlen($originalMessage)]); - $command['message'] = FailureText::bound($originalMessage, + $command['message'] = $this->text($originalMessage, $messageLimit - strlen($suffix), 'message', true)['value'].$suffix; } @@ -181,7 +227,7 @@ private function command(array $command, int $messageLimit, int $stackLimit, boo $report['details'] = $extra; } $innerMessage = array_key_exists('message', $exception) - ? FailureText::bound((string) $exception['message'], $messageLimit, 'exception.message') : null; + ? $this->text((string) $exception['message'], $messageLimit, 'exception.message') : null; $additionalChanges = $innerMessage !== null && $innerMessage['changed'] ? ['exception.message' => ['original_bytes' => $innerMessage['original_bytes']]] : []; $bounded = $this->report($report, $messageLimit, $stackLimit, $dropMetadata, 'diagnostics', $additionalChanges); diff --git a/src/Failure/FailureText.php b/src/Failure/FailureText.php index 1c602253..9764a36d 100644 --- a/src/Failure/FailureText.php +++ b/src/Failure/FailureText.php @@ -16,7 +16,7 @@ public static function bound(string $original, int $limit, string $field, bool $ if (!$forceMarker && $bytes <= $limit && preg_match('//u', $original) === 1) { return ['value' => $original, 'changed' => false, 'original_bytes' => $bytes, 'retained_bytes' => $bytes]; } - $marker = "\n[truncated failure report; original {$field} {$bytes} bytes]"; + $marker = self::marker($original, $field); $available = $limit - strlen($marker); if ($available < 0) { throw new FailureReportRefused($limit); @@ -29,7 +29,7 @@ public static function bound(string $original, int $limit, string $field, bool $ } if (preg_match('//u', $candidate) !== 1) { // Repair only the bounded prefix, never allocate an escaped copy of - // an arbitrarily large original just to retain its first64KiB. + // an arbitrarily large original just to retain its first 64 KiB. $candidate = json_decode(json_encode($prefix, JSON_THROW_ON_ERROR | JSON_INVALID_UTF8_SUBSTITUTE | JSON_UNESCAPED_UNICODE), true, flags: JSON_THROW_ON_ERROR); $candidate = substr($candidate, 0, $available); @@ -41,4 +41,15 @@ public static function bound(string $original, int $limit, string $field, bool $ return ['value' => $candidate.$marker, 'changed' => true, 'original_bytes' => $bytes, 'retained_bytes' => strlen($candidate)]; } + public static function minimumBytes(string $original, string $field): int + { + return strlen(self::marker($original, $field)); + } + + private static function marker(string $original, string $field): string + { + $bytes = strlen($original); + return "\n[truncated failure report; original {$field} {$bytes} bytes]"; + } + } diff --git a/src/Worker.php b/src/Worker.php index e2bc3231..51a4de1d 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -2540,7 +2540,8 @@ function (Throwable $failure) use ($taskId, $attemptId, $leaseOwner): void { $failure->getMessage(), $failure instanceof ActivityExecutionFailure ? $failure->originalType : $failure::class, $failure instanceof ActivityCancelled || ($failure instanceof ActivityExecutionFailure && $failure->cancelled), - stackTrace: $failure->getTraceAsString(), + stackTrace: $failure instanceof ActivityExecutionFailure ? $failure->originalStackTrace : $failure->getTraceAsString(), + diagnostics: $failure instanceof ActivityExecutionFailure ? $failure->originalDiagnostics : null, ); }, ); diff --git a/src/Worker/ActivityExecutionFailure.php b/src/Worker/ActivityExecutionFailure.php index 9c499138..9f4805c6 100644 --- a/src/Worker/ActivityExecutionFailure.php +++ b/src/Worker/ActivityExecutionFailure.php @@ -9,6 +9,7 @@ /** @internal Failure metadata returned by an isolated activity callback. */ final class ActivityExecutionFailure extends RuntimeException { + /** @param array|null $originalDiagnostics */ public function __construct( string $message, public readonly string $originalType, @@ -17,6 +18,8 @@ public function __construct( public readonly bool $invalidReport = false, public readonly ?string $timeoutKind = null, public readonly bool $storageAdmissionFailure = false, + public readonly ?string $originalStackTrace = null, + public readonly ?array $originalDiagnostics = null, ) { parent::__construct($message); } diff --git a/src/Worker/CooperativeActivityExecutor.php b/src/Worker/CooperativeActivityExecutor.php index 159cc96f..2eaa8df7 100644 --- a/src/Worker/CooperativeActivityExecutor.php +++ b/src/Worker/CooperativeActivityExecutor.php @@ -7,6 +7,10 @@ use Closure; use DurableWorkflow\Codec\PayloadCodec; use DurableWorkflow\Exception\ActivityCancelled; +use DurableWorkflow\Exception\FailureReportRefused; +use DurableWorkflow\Failure\FailureReportLimiter; +use DurableWorkflow\Failure\FailureReportLimits; +use DurableWorkflow\Transport\JsonDocument; use DurableWorkflow\Exception\InvalidLocalActivityReport; use DurableWorkflow\Exception\LocalActivityTimedOut; use DurableWorkflow\Exception\ServerException; @@ -97,7 +101,7 @@ public function execute(Closure $callback, Closure $heartbeat, Closure $check, ? throw new WorkflowClaimAborted('Activity IPC closed before a fenced result.'); } $buffer .= $chunk; - if (strlen($buffer) > $this->maxFrameBytes + 4) { + if (strlen($buffer) > $this->receiveFrameBytes() + 4) { throw new WorkflowClaimAborted('Activity IPC exceeded its finite frame bound.'); } continue; @@ -131,6 +135,9 @@ public function execute(Closure $callback, Closure $heartbeat, Closure $check, ? $check(true); return $this->decode($message); + case 'report_refused': + throw new WorkflowClaimAborted('failure_report_refused: the original callback failure cannot be encoded.', + previous: new FailureReportRefused(FailureReportLimits::REQUEST_BYTES)); case 'failure': $check(true); if (!is_string($message['message'] ?? null) || !is_string($message['type'] ?? null) @@ -138,6 +145,8 @@ public function execute(Closure $callback, Closure $heartbeat, Closure $check, ? || !is_bool($message['cancelled'] ?? null) || !is_bool($message['invalid_report'] ?? null) || !is_bool($message['storage_admission_failure'] ?? null) + || (isset($message['stack_trace']) && !is_string($message['stack_trace'])) + || (isset($message['diagnostics']) && !is_array($message['diagnostics'])) || !array_key_exists('timeout_kind', $message) || ($message['timeout_kind'] !== null && !in_array($message['timeout_kind'], ['heartbeat', 'start_to_close', 'schedule_to_close'], true))) { @@ -145,7 +154,7 @@ public function execute(Closure $callback, Closure $heartbeat, Closure $check, ? } throw new ActivityExecutionFailure($message['message'], $message['type'], $message['encoding'], $message['cancelled'], $message['invalid_report'], $message['timeout_kind'], - $message['storage_admission_failure']); + $message['storage_admission_failure'], $message['stack_trace'] ?? null, $message['diagnostics'] ?? null); default: throw new WorkflowClaimAborted('Activity IPC returned an unexpected message.'); } @@ -257,7 +266,7 @@ public function executeConcurrent(array $operations, Closure $settled): void throw new WorkflowClaimAborted('Group activity IPC closed before a fenced result.'); } $process->buffer .= $chunk; - if (strlen($process->buffer) > $this->maxFrameBytes + 4) { + if (strlen($process->buffer) > $this->receiveFrameBytes() + 4) { throw new WorkflowClaimAborted('Group activity IPC exceeded its finite frame bound.'); } } @@ -265,6 +274,10 @@ public function executeConcurrent(array $operations, Closure $settled): void while (($message = $this->takeFrame($process->buffer)) !== null) { $operation = $operations[$index]; $kind = $message['kind']; + if ($kind === 'report_refused') { + throw new WorkflowClaimAborted('failure_report_refused: the original callback failure cannot be encoded.', + previous: new FailureReportRefused(FailureReportLimits::REQUEST_BYTES)); + } if ($kind === 'started') { if ($process->callbackPid !== null || !is_int($message['callback_pid'] ?? null) || $message['callback_pid'] < 1) { throw new WorkflowClaimAborted('Group activity IPC returned an invalid callback PID.'); @@ -365,6 +378,8 @@ private function failure(array $message): ActivityExecutionFailure if (!is_string($message['message'] ?? null) || !is_string($message['type'] ?? null) || !is_bool($message['encoding'] ?? null) || !is_bool($message['cancelled'] ?? null) || !is_bool($message['invalid_report'] ?? null) || !is_bool($message['storage_admission_failure'] ?? null) + || (isset($message['stack_trace']) && !is_string($message['stack_trace'])) + || (isset($message['diagnostics']) && !is_array($message['diagnostics'])) || !array_key_exists('timeout_kind', $message) || ($message['timeout_kind'] !== null && !in_array($message['timeout_kind'], ['heartbeat', 'start_to_close', 'schedule_to_close'], true))) { @@ -372,7 +387,8 @@ private function failure(array $message): ActivityExecutionFailure } return new ActivityExecutionFailure($message['message'], $message['type'], $message['encoding'], - $message['cancelled'], $message['invalid_report'], $message['timeout_kind'], $message['storage_admission_failure']); + $message['cancelled'], $message['invalid_report'], $message['timeout_kind'], $message['storage_admission_failure'], + $message['stack_trace'] ?? null, $message['diagnostics'] ?? null); } /** @param resource $owner @@ -493,25 +509,25 @@ private function callback($socket, Closure $callback): never $this->writeFrame($socket, ['kind' => 'result', 'value' => $this->codec->envelope($result)]); } catch (Throwable $error) { try { - $message = $error->getMessage(); - $type = $error::class; - $invalid = $error instanceof InvalidLocalActivityReport; + $invalid = $error instanceof InvalidLocalActivityReport || preg_match('//u', $error->getMessage()) !== 1; + $body = (new FailureReportLimiter(new FailureReportLimits()))->activityEnvelope([ + 'kind' => 'failure', + 'failure' => ['message' => $error->getMessage(), 'type' => $error::class, + 'stack_trace' => $error->getTraceAsString()], + 'encoding' => $encoding, 'cancelled' => !$invalid && $error instanceof ActivityCancelled, + 'invalid_report' => $invalid, + 'timeout_kind' => !$invalid && $error instanceof LocalActivityTimedOut ? $error->timeoutKind : null, + 'storage_admission_failure' => $error instanceof ServerException && $error->isStorageAdmissionFailure(), + ]); + $failure = $body['failure']; + unset($body['failure']); + $this->writeFrame($socket, $failure + $body); + } catch (FailureReportRefused) { try { - json_encode(['message' => $message, 'type' => $type], JSON_THROW_ON_ERROR); + $this->writeFrame($socket, ['kind' => 'report_refused']); } catch (Throwable) { - $invalid = true; - $message = 'Activity failure metadata could not be encoded for the HTTP JSON wire boundary.'; + // The owning worker has abandoned this claim. } - if (strlen($type) > 255) { - $invalid = true; - $message = 'Activity failure metadata exceeded the published exception type limit.'; - } - $this->writeFrame($socket, ['kind' => 'failure', 'message' => $message, - 'type' => $invalid ? InvalidLocalActivityReport::class : $type, 'encoding' => $encoding, - 'cancelled' => !$invalid && $error instanceof ActivityCancelled, - 'invalid_report' => $invalid, - 'timeout_kind' => !$invalid && $error instanceof LocalActivityTimedOut ? $error->timeoutKind : null, - 'storage_admission_failure' => $error instanceof ServerException && $error->isStorageAdmissionFailure()]); } catch (Throwable) { // The owning worker has abandoned this claim or the frame is invalid. } @@ -571,8 +587,9 @@ private function socketPair(): array */ private function writeFrame($socket, array $message): void { - $body = json_encode($message, JSON_THROW_ON_ERROR); - if (strlen($body) > $this->maxFrameBytes) { + $failure = in_array($message['kind'] ?? null, ['failure', 'report_refused'], true); + $body = $failure ? JsonDocument::encode($message) : json_encode($message, JSON_THROW_ON_ERROR); + if (strlen($body) > ($failure ? FailureReportLimits::REQUEST_BYTES : $this->maxFrameBytes)) { throw new WorkflowClaimAborted('Activity IPC exceeded its finite frame bound.'); } $frame = pack('N', strlen($body)).$body; @@ -611,6 +628,11 @@ private function requireMessage($socket, string $kind): array return $message; } + private function receiveFrameBytes(): int + { + return max($this->maxFrameBytes, FailureReportLimits::REQUEST_BYTES); + } + /** @return array|null */ private function takeFrame(string &$buffer): ?array { @@ -619,7 +641,7 @@ private function takeFrame(string &$buffer): ?array } $prefix = unpack('Nlength', substr($buffer, 0, 4)); $length = $prefix['length'] ?? 0; - if (!is_int($length) || $length < 1 || $length > $this->maxFrameBytes) { + if (!is_int($length) || $length < 1 || $length > $this->receiveFrameBytes()) { throw new WorkflowClaimAborted('Activity IPC received an invalid frame length.'); } if (strlen($buffer) < $length + 4) { @@ -635,6 +657,11 @@ private function takeFrame(string &$buffer): ?array throw new WorkflowClaimAborted('Activity IPC received a malformed message.'); } + $failure = in_array($message['kind'], ['failure', 'report_refused'], true); + if ($length > ($failure ? FailureReportLimits::REQUEST_BYTES : $this->maxFrameBytes)) { + throw new WorkflowClaimAborted('Activity IPC exceeded its original payload frame bound.'); + } + return $message; } diff --git a/tests/CooperativeActivityExecutorTest.php b/tests/CooperativeActivityExecutorTest.php index aece57f1..c8289052 100644 --- a/tests/CooperativeActivityExecutorTest.php +++ b/tests/CooperativeActivityExecutorTest.php @@ -572,6 +572,34 @@ static function (): void {}, } } + public function testOversizedFailureIsIndependentOfTheBusinessPayloadFrameBudget(): void + { + try { + (new CooperativeActivityExecutor(new AvroPayloadCodec(), maxPayloadBytes: 1024))->execute( + static function (): never { throw new RuntimeException(str_repeat('m', 3 * 1024 * 1024)); }, + static fn (array $details): mixed => null, + static function (): void {}, + ); + self::fail('Callback failure was accepted as success.'); + } catch (ActivityExecutionFailure $error) { + self::assertSame(RuntimeException::class, $error->originalType); + self::assertFalse($error->duringEncoding); + self::assertFalse($error->cancelled); + self::assertFalse($error->invalidReport); + self::assertStringContainsString('[truncated failure report;', $error->getMessage()); + self::assertLessThanOrEqual(65536, strlen($error->getMessage())); + self::assertStringContainsString('CooperativeActivityExecutorTest.php', $error->originalStackTrace); + self::assertSame(3145728, $error->originalDiagnostics['truncation']['fields']['message']['original_bytes']); + $bounded = (new \DurableWorkflow\Failure\FailureReportLimiter( + new \DurableWorkflow\Failure\FailureReportLimits(4096)))->activityEnvelope([ + 'failure' => ['message' => $error->getMessage(), 'type' => $error->originalType, + 'stack_trace' => $error->originalStackTrace, 'diagnostics' => $error->originalDiagnostics], + ]); + self::assertLessThanOrEqual(4096, \DurableWorkflow\Transport\JsonDocument::size($bounded)); + self::assertSame(3145728, $bounded['failure']['diagnostics']['truncation']['fields']['message']['original_bytes']); + } + } + public function testResultEncodingFailureIsDistinctFromCallbackFailure(): void { try { diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php index db3a1198..c43cfc7e 100644 --- a/tests/FailureReportTest.php +++ b/tests/FailureReportTest.php @@ -142,6 +142,23 @@ public function testLowerConfiguredLimitCountsEscapingAfterAllMarkers(): void self::assertArrayNotHasKey('details', $body['failure']); } + public function testSmallTransportBudgetRetainsMarkersInsteadOfAnArbitraryContentFloor(): void + { + foreach (['activity', 'workflow'] as $kind) { + $transport = new FakeTransport([[]]); + $client = (new Client('http://recording.invalid', transport: $transport))->withFailureReportLimit(512); + $large = str_repeat('m', 3 * 1024 * 1024); + if ($kind === 'activity') { + $client->failActivityTask('task', 'attempt', 'owner', $large, stackTrace: $large); + } else { + $client->failWorkflowTask('task', 'owner', 1, $large, stackTrace: $large); + } + self::assertCount(1, $transport->requests); + self::assertLessThanOrEqual(512, JsonDocument::size($transport->requests[0]['body'])); + self::assertStringContainsString('[truncated failure report;', $transport->requests[0]['body']['failure']['message']); + } + } + public function testKnownTransportLimitIsRespectedAndImpossibleIdentityRefusesBeforeSend(): void { $transport = new class implements Transport, RequestSizeLimitTransport { From dc6e83e8c3986740a3c4e1c12432b145b3606fd6 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:23:56 +0000 Subject: [PATCH 04/19] Extract failure budget stages and cover isolated callback delivery --- scripts/qualify-failure-report-delivery.php | 64 +++++++++++++++-- src/Failure/FailureReportLimiter.php | 80 +++++++++++++-------- 2 files changed, 109 insertions(+), 35 deletions(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index 99de7498..d0344288 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -13,6 +13,8 @@ final class FailureDeliveryQualificationException extends RuntimeException {} +const FAILURE_DELIVERY_MARKER = '[truncated failure report;'; + function requireDelivery(bool $condition, string $message): void { if (!$condition) { @@ -22,13 +24,19 @@ function requireDelivery(bool $condition, string $message): void $http = new class implements Psr\Http\Client\ClientInterface { public array $reports = []; + public array $claims = []; private GuzzleHttp\Client $client; public function __construct() { $this->client = new GuzzleHttp\Client(['http_errors' => false, 'timeout' => 15]); } public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Http\Message\ResponseInterface + { + $response = $this->client->sendRequest($request); + $this->observe($request, $response); + return $response; + } + public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Message\ResponseInterface $response): void { $wire = (string) $request->getBody(); $data = $wire === '' ? [] : json_decode($wire, true, flags: JSON_THROW_ON_ERROR); - $response = $this->client->sendRequest($request); if ($response->getStatusCode() >= 400) { fwrite(STDERR, json_encode(['path' => $request->getUri()->getPath(), 'status' => $response->getStatusCode(), 'response' => array_diff_key(json_decode((string) $response->getBody(), true), ['server_capabilities' => true])], JSON_THROW_ON_ERROR)."\n"); @@ -41,11 +49,19 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt 'workflow_task_attempt' => $data['workflow_task_attempt'] ?? null, 'type' => isset($data['failure']) ? ($failure['type'] ?? null) : ($failure['exception_type'] ?? null), 'message_bytes' => strlen($failure['message']), - 'marked' => str_contains($failure['message'], '[truncated failure report;'), + 'marked' => str_contains($failure['message'], FAILURE_DELIVERY_MARKER), 'stack_bytes' => strlen($failure['stack_trace'] ?? $failure['exception']['stack_trace'] ?? ''), 'message_sha256' => hash('sha256', $failure['message']), 'non_retryable' => $failure['non_retryable'] ?? null]; } - return $response; + if (str_ends_with($path, '/poll')) { + $reply = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + $task = $reply['task'] ?? null; + if (is_array($task)) { + $this->claims[] = array_intersect_key($task, array_flip([ + 'task_id', 'lease_owner', 'workflow_task_attempt', 'activity_attempt_id', 'attempt_id', 'workflow_id', 'run_id', + ])); + } + } } }; $namespace = 'php-failure-'.bin2hex(random_bytes(6)); @@ -70,10 +86,11 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt return ['fresh_turn' => true]; } try { - $context->activity('qualification.failure.activity', [], ['retry_policy' => ['max_attempts' => 1]]); + $context->activity($kind === 'cooperative-worker' ? 'qualification.failure.managed' : 'qualification.failure.activity', + [], ['retry_policy' => ['max_attempts' => 1]]); throw new FailureDeliveryQualificationException('The activity unexpectedly succeeded.'); } catch (ActivityFailed $failure) { - return ['type' => $failure->failureType, 'marked' => str_contains($failure->getMessage(), '[truncated failure report;'), + return ['type' => $failure->failureType, 'marked' => str_contains($failure->getMessage(), FAILURE_DELIVERY_MARKER), 'message_bytes' => strlen($failure->getMessage()), 'non_retryable' => $failure->nonRetryable]; } }); @@ -86,6 +103,7 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt register_shutdown_function(static function () use (&$result, $http): void { if (!isset($result['pass'])) { $result['reports'] = $http->reports; + $result['claims'] = $http->claims; $result['incomplete'] = true; $result['last_error'] = error_get_last(); echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; @@ -119,6 +137,39 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt 'task_id' => $task['task_id'], 'attempt_id' => $task['activity_attempt_id'] ?? $task['attempt_id'], 'lease_owner' => $task['lease_owner'], 'output' => $output, 'event_counts' => $counts]; } +$stack = GuzzleHttp\HandlerStack::create(); +$stack->push(static function (callable $next) use ($http): Closure { + return static function (Psr\Http\Message\RequestInterface $request, array $options) use ($next, $http) { + return $next($request, $options)->then(static function (Psr\Http\Message\ResponseInterface $response) use ($request, $http) { + $http->observe($request, $response); + return $response; + }); + }; +}); +$managedId = $namespace.'-cooperative'; +$managedClient = new Client($url, namespace: $namespace, workerToken: 'claimed-task-worker', + transport: new DurableWorkflow\Transport\Psr18Transport(client: new GuzzleHttp\Client(['handler' => $stack, 'http_errors' => false]))); +$managedWorker = new Worker($managedClient, $queue, workerId: $managedId, enableCooperativeCancellation: true); +$managedWorker->registerActivity('qualification.failure.managed', static function (): never { + throw new FailureDeliveryQualificationException(str_repeat('m', 3 * 1024 * 1024)); +}); +$managedClient->registerWorker($managedId, $queue, [], ['qualification.failure.managed'], ['cooperative_cancellation']); +$handle = $control->startWorkflow('qualification.failure', $namespace.'-cooperative-worker', $queue, ['cooperative-worker'], runTimeoutSeconds: 120); +$worker->tick(0); +$managedWorker->tick(0); +$worker->tick(0); +$output = $handle->resultOfSelectedRun(15, 0.05); +requireDelivery($output['type'] === FailureDeliveryQualificationException::class && $output['marked'] === true, + 'The original workflow did not receive its isolated callback failure.'); +requireDelivery($output['non_retryable'] === false, 'The callback retry classification changed.'); +$history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); +$events = $history['events'] ?? $history['history_events'] ?? []; +$failed = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityFailed')); +$failureWire = json_encode($failed, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); +requireDelivery(str_contains($failureWire, '3145728') && str_contains($failureWire, 'qualify-failure-report-delivery.php'), + 'Original callback trace or original-size diagnostics missing from history.'); +$result['scenarios']['cooperative-worker'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'output' => $output, 'original_callback_trace_and_sizes_in_history' => true]; $handle = $control->startWorkflow('qualification.failure', $namespace.'-retry', $queue, ['retry'], runTimeoutSeconds: 120); $worker->tick(0); $original = $client->pollActivityTask($activityWorkerId, $queue, 0); @@ -174,7 +225,7 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt throw new FailureDeliveryQualificationException('Terminal workflow unexpectedly succeeded.'); } catch (WorkflowFailed $failure) { requireDelivery($failure->failureType === FailureDeliveryQualificationException::class, 'Workflow failure type changed.'); - requireDelivery(str_contains($failure->getMessage(), '[truncated failure report;'), 'Terminal workflow marker missing.'); + requireDelivery(str_contains($failure->getMessage(), FAILURE_DELIVERY_MARKER), 'Terminal workflow marker missing.'); $result['scenarios']['terminal'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, 'failure_type' => $failure->failureType, 'marked_message' => true, 'message_bytes' => strlen($failure->getMessage())]; } @@ -182,5 +233,6 @@ public function sendRequest(Psr\Http\Message\RequestInterface $request): Psr\Htt requireDelivery($report['encoded_bytes'] <= 1048576 && $report['http_status'] === 200, 'Encoded/reporting HTTP boundary failed.'); } $result['reports'] = $http->reports; +$result['claims'] = $http->claims; $result['pass'] = true; echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index 1fba830e..08965702 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -32,22 +32,28 @@ public function request(string $path, array $body): array if (!$endpoint && !$commands) { return $body; } + $carrier = $endpoint ? match ($matches[1]) { + 'activity' => 'diagnostics', + default => 'stack_trace', + } : null; + return $this->bound($body, $carrier, $commands, $this->effectiveLimit(), strlen($path)); + } + + private function effectiveLimit(): int + { $limit = $this->limits->requestBytes; - if ($this->transport instanceof RequestSizeLimitTransport) { - try { - $known = $this->transport->maximumRequestBytes(); - } catch (\Throwable $error) { - throw new FailureReportRefused($limit, $error); - } - if ($known !== null) { - if ($known < 1) { - throw new FailureReportRefused($limit); - } - $limit = min($limit, $known); - } + if (!$this->transport instanceof RequestSizeLimitTransport) { + return $limit; + } + try { + $known = $this->transport->maximumRequestBytes(); + } catch (\Throwable $error) { + throw new FailureReportRefused($limit, $error); + } + if ($known !== null && $known < 1) { + throw new FailureReportRefused($limit); } - return $this->bound($body, $endpoint ? ($matches[1] === 'activity' ? 'diagnostics' : 'stack_trace') : null, - $commands, $limit, strlen($path)); + return $known === null ? $limit : min($limit, $known); } /** @internal The callback IPC carries the same bounded report before HTTP admission. @@ -74,11 +80,7 @@ private function bound(array $body, ?string $carrier, bool $commands, int $limit $dropMetadata, $carrier); } if ($commands) { - foreach ($body['commands'] as $index => $command) { - if (is_array($command) && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) { - $candidate['commands'][$index] = $this->command($command, $messageLimit, $stackLimit, $dropMetadata); - } - } + $candidate['commands'] = $this->commands($body['commands'], $messageLimit, $stackLimit, $dropMetadata); } try { if (JsonDocument::size($candidate) + $pathBytes <= $limit) { @@ -197,22 +199,26 @@ private function omitMetadata(array &$report, array $metadata, array &$changes): } } + /** @param array $commands + * @return array + */ + private function commands(array $commands, int $messageLimit, int $stackLimit, bool $dropMetadata): array + { + foreach ($commands as $index => $command) { + if (is_array($command) && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) { + $commands[$index] = $this->command($command, $messageLimit, $stackLimit, $dropMetadata); + } + } + return $commands; + } + /** @param array $command * @return array */ private function command(array $command, int $messageLimit, int $stackLimit, bool $dropMetadata): array { if (($command['type'] ?? null) === 'fail_update') { - $originalMessage = (string) ($command['message'] ?? ''); - $bounded = $this->text($originalMessage, $messageLimit, 'message'); - $command['message'] = $bounded['value']; - if ($bounded['changed']) { - $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['original_message_bytes' => strlen($originalMessage)]); - $command['message'] = $this->text($originalMessage, - $messageLimit - strlen($suffix), 'message', true)['value'].$suffix; - } - - return $command; + return $this->updateCommand($command, $messageLimit); } $exception = is_array($command['exception'] ?? null) ? $command['exception'] : []; $fixed = array_intersect_key($exception, array_flip(['class', 'type', 'code', 'line'])); @@ -247,4 +253,20 @@ private function command(array $command, int $messageLimit, int $stackLimit, boo return $command; } + /** @param array $command + * @return array + */ + private function updateCommand(array $command, int $messageLimit): array + { + $originalMessage = (string) ($command['message'] ?? ''); + $bounded = $this->text($originalMessage, $messageLimit, 'message'); + $command['message'] = $bounded['value']; + if ($bounded['changed']) { + $suffix = "\n[truncation diagnostics]\n".JsonDocument::encode(['original_message_bytes' => strlen($originalMessage)]); + $command['message'] = $this->text($originalMessage, + $messageLimit - strlen($suffix), 'message', true)['value'].$suffix; + } + return $command; + } + } From f8faf6ea2d7c6bd3fc1792ef69e1ee134d4de02b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:24:11 +0000 Subject: [PATCH 05/19] Declare the qualification activity context explicitly --- scripts/qualify-failure-report-delivery.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index d0344288..59d389f6 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -150,7 +150,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $managedClient = new Client($url, namespace: $namespace, workerToken: 'claimed-task-worker', transport: new DurableWorkflow\Transport\Psr18Transport(client: new GuzzleHttp\Client(['handler' => $stack, 'http_errors' => false]))); $managedWorker = new Worker($managedClient, $queue, workerId: $managedId, enableCooperativeCancellation: true); -$managedWorker->registerActivity('qualification.failure.managed', static function (): never { +$managedWorker->registerActivity('qualification.failure.managed', static function (DurableWorkflow\Worker\ActivityContext $_context): never { throw new FailureDeliveryQualificationException(str_repeat('m', 3 * 1024 * 1024)); }); $managedClient->registerWorker($managedId, $queue, [], ['qualification.failure.managed'], ['cooperative_cancellation']); From 907a466a54ef83719775981606f29ce63cb6490b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:33:16 +0000 Subject: [PATCH 06/19] Reserve summary headroom and freeze local failure observations before branching --- docs/portal/build/failures-retries.md | 6 +- scripts/qualify-failure-report-delivery.php | 31 +++++++- src/Client.php | 12 ++- src/Failure/FailureReportLimiter.php | 34 +++++++-- src/Failure/FailureReportLimits.php | 2 +- src/Failure/FailureText.php | 8 +- src/Failure/LocalFailureReports.php | 85 +++++++++++++++++++++ src/Worker.php | 11 ++- src/Worker/WorkflowCommand.php | 5 +- tests/CooperativeActivityExecutorTest.php | 2 +- tests/FailureReportTest.php | 2 +- tests/LocalFailureReportTest.php | 85 +++++++++++++++++++++ 12 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 src/Failure/LocalFailureReports.php create mode 100644 tests/LocalFailureReportTest.php diff --git a/docs/portal/build/failures-retries.md b/docs/portal/build/failures-retries.md index 7c993e06..6dea5123 100644 --- a/docs/portal/build/failures-retries.md +++ b/docs/portal/build/failures-retries.md @@ -114,11 +114,15 @@ Failure reporting keeps diagnostic content within finite defaults: | Content | Default budget | | --- | --- | -| UTF-8 message, including its marker | 64 KiB | +| UTF-8 message, including its marker | 62 KiB | | Stack trace, including diagnostic markers | 128 KiB | | Serialized details and diagnostics together | 64 KiB | | Complete encoded failure request, including JSON escaping and identity | 1 MiB | +The message budget reserves 2 KiB below the original 64 KiB ceiling for +published runtime summaries. It does not repair the separate runtime summary +byte-bound defect tracked in shared issue 225. + Oversized content produces a visible `[truncated failure report; ...]` message marker and bounded diagnostics with original sizes and omitted fields. The SDK retains a complete valid Avro details envelope or explicitly omits it. It never diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index 59d389f6..f2bb9181 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -74,7 +74,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $workerId = $namespace.'-workflow'; $activityWorkerId = $namespace.'-activity'; $worker = new Worker($client, $queue, workerId: $workerId, stickyCacheCapacity: 1); -$worker->registerWorkflow('qualification.failure', static function (WorkflowContext $context, string $kind): array { +$workflowHandler = static function (WorkflowContext $context, string $kind): array { if ($kind === 'terminal') { throw new FailureDeliveryQualificationException(str_repeat('t', 3 * 1024 * 1024)); } @@ -86,13 +86,22 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes return ['fresh_turn' => true]; } try { + if ($kind === 'local-failure') { + $context->localActivity('qualification.failure.local', [], ['retry_policy' => ['max_attempts' => 1]]); + } else { $context->activity($kind === 'cooperative-worker' ? 'qualification.failure.managed' : 'qualification.failure.activity', [], ['retry_policy' => ['max_attempts' => 1]]); + } throw new FailureDeliveryQualificationException('The activity unexpectedly succeeded.'); } catch (ActivityFailed $failure) { return ['type' => $failure->failureType, 'marked' => str_contains($failure->getMessage(), FAILURE_DELIVERY_MARKER), - 'message_bytes' => strlen($failure->getMessage()), 'non_retryable' => $failure->nonRetryable]; + 'message_bytes' => strlen($failure->getMessage()), 'message_sha256' => hash('sha256', $failure->getMessage()), + 'non_retryable' => $failure->nonRetryable]; } +}; +$worker->registerWorkflow('qualification.failure', $workflowHandler); +$worker->registerActivity('qualification.failure.local', static function (DurableWorkflow\Worker\ActivityContext $_context): never { + throw new FailureDeliveryQualificationException(str_repeat('雪', 1024 * 1024)); }); $client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes'], workflowCommandContracts: $worker->contracts()['workflow_commands']); @@ -137,6 +146,24 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'task_id' => $task['task_id'], 'attempt_id' => $task['activity_attempt_id'] ?? $task['attempt_id'], 'lease_owner' => $task['lease_owner'], 'output' => $output, 'event_counts' => $counts]; } +$handle = $control->startWorkflow('qualification.failure', $namespace.'-local', $queue, ['local-failure'], runTimeoutSeconds: 120); +$worker->tick(0); +$output = $handle->resultOfSelectedRun(15, 0.05); +$history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); +$events = $history['events'] ?? $history['history_events'] ?? []; +$failed = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityFailed')); +requireDelivery(count($failed) === 1, 'Local failure was not persisted once.'); +$message = $failed[0]['payload']['message'] ?? ''; +requireDelivery($output['message_sha256'] === hash('sha256', $message), 'Initial local branch saw a different failure from persisted history.'); +requireDelivery($output['type'] === FailureDeliveryQualificationException::class && $output['marked'] === true, + 'Original local failure type or marker changed.'); +$cold = (new DurableWorkflow\Worker\Replayer($client->payloadCodec()))->replay($workflowHandler, $events, ['local-failure'], $queue, + localActivityExecutor: static function (): never { throw new FailureDeliveryQualificationException('Cold replay repeated a local callback.'); }); +requireDelivery($cold->terminalFailure === null && count($cold->commands) === 1, 'Local cold replay did not complete normally.'); +requireDelivery($client->payloadCodec()->decodeEnvelope($cold->commands[0]['result']) === $output, + 'Cold workflow branch saw a different local failure.'); +$result['scenarios']['local-failure'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, + 'output' => $output, 'initial_equals_persisted_equals_cold' => true]; $stack = GuzzleHttp\HandlerStack::create(); $stack->push(static function (callable $next) use ($http): Closure { return static function (Psr\Http\Message\RequestInterface $request, array $options) use ($next, $http) { diff --git a/src/Client.php b/src/Client.php index b12a0552..71fda799 100644 --- a/src/Client.php +++ b/src/Client.php @@ -12,6 +12,7 @@ use DurableWorkflow\Exception\FailureReportRefused; use DurableWorkflow\Failure\FailureReportLimiter; use DurableWorkflow\Failure\FailureReportLimits; +use DurableWorkflow\Failure\LocalFailureReports; use DurableWorkflow\Exception\QueryFailed; use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Exception\SignalFailed; @@ -151,6 +152,15 @@ public function withFailureReportLimit(int $requestBytes): self return $copy; } + /** @internal Freeze local diagnostic values before workflow code branches on them. + * @param array $outcome + * @return array + */ + public function normalizeLocalFailure(array $outcome): array + { + return LocalFailureReports::bound($outcome, $this->failureReports->effectiveLimit()); + } + public function payloadCodec(): PayloadCodec { return $this->codec; @@ -2640,7 +2650,7 @@ private function request( $pollSeconds = $poll ? max(0, min(60, $body['timeout_seconds'])) : 0; $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; if ($body !== null) { - $body = $this->failureReports->request($path, $body); + $body = $this->failureReports->request($path, $body, false); $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget); $body = $this->failureReports->request($path, $body); } diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index 08965702..4beffa14 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -21,14 +21,18 @@ public function __construct(private readonly FailureReportLimits $limits, privat /** @param array $body * @return array */ - public function request(string $path, array $body): array + public function request(string $path, array $body, bool $enforceRequestLimit = true): array { + if (preg_match('~^/worker/workflow-tasks/[^/]+/local-activities/[^/]+/outcome$~D', $path) === 1 + && is_array($body['report'] ?? null) && is_string($body['report']['message'] ?? null)) { + return $this->preparedOutcome($path, $body); + } $endpoint = preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path, $matches) === 1 && is_array($body['failure'] ?? null); $commands = preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 && is_array($body['commands'] ?? null) && array_filter($body['commands'], static fn (mixed $command): bool => is_array($command) - && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) !== []; + && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update', 'record_local_activity'], true)) !== []; if (!$endpoint && !$commands) { return $body; } @@ -36,10 +40,27 @@ public function request(string $path, array $body): array 'activity' => 'diagnostics', default => 'stack_trace', } : null; - return $this->bound($body, $carrier, $commands, $this->effectiveLimit(), strlen($path)); + return $this->bound($body, $carrier, $commands, $this->effectiveLimit(), strlen($path), $enforceRequestLimit); + } + + /** @param array $body + * @return array + */ + private function preparedOutcome(string $path, array $body): array + { + $limit = $this->effectiveLimit(); + $fixed = $body; + $fixed['report'] = []; + $available = $limit - JsonDocument::size($fixed) - strlen($path); + if ($available < 1) { + throw new FailureReportRefused($limit); + } + $body['report'] = LocalFailureReports::bound($body['report'], $available); + return $body; } - private function effectiveLimit(): int + /** @internal Known body budget shared with pre-observation local normalization. */ + public function effectiveLimit(): int { $limit = $this->limits->requestBytes; if (!$this->transport instanceof RequestSizeLimitTransport) { @@ -68,7 +89,7 @@ public function activityEnvelope(array $body): array /** @param array $body * @return array */ - private function bound(array $body, ?string $carrier, bool $commands, int $limit, int $pathBytes): array + private function bound(array $body, ?string $carrier, bool $commands, int $limit, int $pathBytes, bool $enforceRequestLimit = true): array { $messageLimit = FailureReportLimits::MESSAGE_BYTES; $stackLimit = FailureReportLimits::STACK_BYTES; @@ -83,7 +104,8 @@ private function bound(array $body, ?string $carrier, bool $commands, int $limit $candidate['commands'] = $this->commands($body['commands'], $messageLimit, $stackLimit, $dropMetadata); } try { - if (JsonDocument::size($candidate) + $pathBytes <= $limit) { + $bytes = JsonDocument::size($candidate); + if (!$enforceRequestLimit || $bytes + $pathBytes <= $limit) { return $candidate; } } catch (\JsonException $error) { diff --git a/src/Failure/FailureReportLimits.php b/src/Failure/FailureReportLimits.php index 7d44b044..4d0d48e4 100644 --- a/src/Failure/FailureReportLimits.php +++ b/src/Failure/FailureReportLimits.php @@ -7,7 +7,7 @@ /** SDK failure-report budgets; these do not change the Server's request limit. */ final class FailureReportLimits { - public const MESSAGE_BYTES = 65536; + public const MESSAGE_BYTES = 63488; public const STACK_BYTES = 131072; public const METADATA_BYTES = 65536; public const REQUEST_BYTES = 1048576; diff --git a/src/Failure/FailureText.php b/src/Failure/FailureText.php index 9764a36d..25275dea 100644 --- a/src/Failure/FailureText.php +++ b/src/Failure/FailureText.php @@ -10,13 +10,13 @@ final class FailureText { /** @return array{value: string, changed: bool, original_bytes: int, retained_bytes: int} */ - public static function bound(string $original, int $limit, string $field, bool $forceMarker = false): array + public static function bound(string $original, int $limit, string $field, bool $forceMarker = false, ?int $originalBytes = null): array { $bytes = strlen($original); if (!$forceMarker && $bytes <= $limit && preg_match('//u', $original) === 1) { return ['value' => $original, 'changed' => false, 'original_bytes' => $bytes, 'retained_bytes' => $bytes]; } - $marker = self::marker($original, $field); + $marker = self::marker($original, $field, $originalBytes); $available = $limit - strlen($marker); if ($available < 0) { throw new FailureReportRefused($limit); @@ -46,9 +46,9 @@ public static function minimumBytes(string $original, string $field): int return strlen(self::marker($original, $field)); } - private static function marker(string $original, string $field): string + private static function marker(string $original, string $field, ?int $originalBytes = null): string { - $bytes = strlen($original); + $bytes = $originalBytes ?? strlen($original); return "\n[truncated failure report; original {$field} {$bytes} bytes]"; } diff --git a/src/Failure/LocalFailureReports.php b/src/Failure/LocalFailureReports.php new file mode 100644 index 00000000..e7751486 --- /dev/null +++ b/src/Failure/LocalFailureReports.php @@ -0,0 +1,85 @@ + $outcome + * @return array + */ + public static function bound(array $outcome, int $requestLimit = FailureReportLimits::REQUEST_BYTES): array + { + $messageLimit = FailureReportLimits::MESSAGE_BYTES; + for ($pass = 0; $pass < 18; ++$pass) { + $bounded = self::fragment($outcome, $messageLimit); + foreach ($outcome['attempts'] ?? [] as $index => $attempt) { + if (is_array($attempt)) { + $bounded['attempts'][$index] = self::fragment($attempt, $messageLimit); + } + } + // A successful result keeps its independent business-payload policy. + $diagnosticContent = $bounded; + unset($diagnosticContent['result']); + try { + if (JsonDocument::size($diagnosticContent) <= $requestLimit) { + return $bounded; + } + } catch (\JsonException $error) { + throw new FailureReportRefused($requestLimit, $error); + } + $messageLimit = intdiv($messageLimit, 2); + } + throw new FailureReportRefused($requestLimit); + } + + /** @param array $fragment + * @return array + */ + private static function fragment(array $fragment, int $limit): array + { + if (!is_string($fragment['message'] ?? null)) { + return $fragment; + } + $message = trim($fragment['message']); + $fragment['message'] = $message; + $originalBytes = self::originalBytes($message); + $existingDiagnostics = str_contains($message, self::DIAGNOSTICS); + if (strlen($message) <= $limit && preg_match('//u', $message) === 1 + && ($originalBytes === strlen($message) || $existingDiagnostics)) { + return $fragment; + } + $suffix = self::DIAGNOSTICS.JsonDocument::encode(['message' => ['original_bytes' => $originalBytes]]); + $minimum = FailureText::minimumBytes($message, 'message') + strlen($suffix) + 24; + $budget = max($limit, $minimum); + $fragment['message'] = FailureText::bound($message, $budget - strlen($suffix), 'message', true, $originalBytes)['value'].$suffix; + return $fragment; + } + + private static function originalBytes(string $message): int + { + $position = strrpos($message, self::DIAGNOSTICS); + if ($position !== false) { + try { + $value = json_decode(substr($message, $position + strlen(self::DIAGNOSTICS)), true, flags: JSON_THROW_ON_ERROR); + $bytes = is_array($value) ? ($value['message']['original_bytes'] ?? null) : null; + if (is_int($bytes) && $bytes >= 0) { + return $bytes; + } + } catch (\JsonException) { + // Caller text resembling a marker remains ordinary text. + } + } + if (preg_match('/\n\[truncated failure report; original message ([0-9]+) bytes\]$/D', $message, $matches) === 1) { + return (int) $matches[1]; + } + return strlen($message); + } +} diff --git a/src/Worker.php b/src/Worker.php index 51a4de1d..143d6ed0 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -3245,9 +3245,9 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi 'heartbeats' => $heartbeats, ]; - return ['outcome' => 'completed', 'result' => $result, 'attempts' => $attempts]; + return $this->client->normalizeLocalFailure(['outcome' => 'completed', 'result' => $result, 'attempts' => $attempts]); } catch (Throwable $exception) { - if ($exception instanceof WorkflowClaimAborted) { + if ($exception instanceof WorkflowClaimAborted || $exception instanceof FailureReportRefused) { throw $exception; } if ($exception instanceof ServerException && $exception->isStorageAdmissionFailure()) { @@ -3336,14 +3336,14 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi continue; } - return [ + return $this->client->normalizeLocalFailure([ 'outcome' => $cancelled ? 'cancelled' : ($timedOut ? 'timed_out' : 'failed'), 'message' => $message, 'exception_type' => $type, 'non_retryable' => $isNonRetryable, ...($timeoutKind === null ? [] : ['timeout_kind' => $timeoutKind]), 'attempts' => $attempts, - ]; + ]); } } @@ -3950,6 +3950,9 @@ private function workflowFailureCommand( ?string $failedActivityExecutionId = null, ): array { + if ($exception instanceof FailureReportRefused) { + throw $exception; + } $command = [ 'type' => 'fail_workflow', 'message' => $exception->getMessage(), diff --git a/src/Worker/WorkflowCommand.php b/src/Worker/WorkflowCommand.php index 5fa42564..b83c435d 100644 --- a/src/Worker/WorkflowCommand.php +++ b/src/Worker/WorkflowCommand.php @@ -11,6 +11,8 @@ use DurableWorkflow\Codec\PayloadCodec; use DurableWorkflow\Exception\InvalidLocalActivityReport; use DurableWorkflow\Exception\ServerException; +use DurableWorkflow\Exception\FailureReportRefused; +use DurableWorkflow\Failure\LocalFailureReports; use InvalidArgumentException; use LogicException; use Throwable; @@ -435,7 +437,7 @@ public function resolveLocalActivity(PayloadCodec $codec): self try { $outcome = ($this->localActivity)($activityType, $arguments, $options); } catch (Throwable $exception) { - if ($exception instanceof WorkflowClaimAborted) { + if ($exception instanceof WorkflowClaimAborted || $exception instanceof FailureReportRefused) { throw $exception; } if ($exception instanceof ServerException && $exception->isStorageAdmissionFailure()) { @@ -492,6 +494,7 @@ public function resolveLocalActivity(PayloadCodec $codec): self $resultEnvelope = null; } + $outcome = LocalFailureReports::bound($outcome); $result = $status === 'completed' ? ($outcome['result'] ?? null) : new \DurableWorkflow\Exception\ActivityFailed( diff --git a/tests/CooperativeActivityExecutorTest.php b/tests/CooperativeActivityExecutorTest.php index c8289052..b33679b9 100644 --- a/tests/CooperativeActivityExecutorTest.php +++ b/tests/CooperativeActivityExecutorTest.php @@ -587,7 +587,7 @@ static function (): void {}, self::assertFalse($error->cancelled); self::assertFalse($error->invalidReport); self::assertStringContainsString('[truncated failure report;', $error->getMessage()); - self::assertLessThanOrEqual(65536, strlen($error->getMessage())); + self::assertLessThanOrEqual(63488, strlen($error->getMessage())); self::assertStringContainsString('CooperativeActivityExecutorTest.php', $error->originalStackTrace); self::assertSame(3145728, $error->originalDiagnostics['truncation']['fields']['message']['original_bytes']); $bounded = (new \DurableWorkflow\Failure\FailureReportLimiter( diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php index c43cfc7e..311341ad 100644 --- a/tests/FailureReportTest.php +++ b/tests/FailureReportTest.php @@ -57,7 +57,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface $failure = $body['failure']; self::assertSame('OriginalFailure', $failure['type']); self::assertTrue($failure['non_retryable']); - self::assertLessThanOrEqual(65536, strlen($failure['message'])); + self::assertLessThanOrEqual(63488, strlen($failure['message'])); self::assertLessThanOrEqual(131072, strlen($failure['stack_trace'] ?? '')); self::assertSame(1, preg_match('//u', $failure['message'])); $metadata = array_intersect_key($failure, array_flip(['details', 'diagnostics', 'runtime_diagnostics'])); diff --git a/tests/LocalFailureReportTest.php b/tests/LocalFailureReportTest.php new file mode 100644 index 00000000..3f6eea4d --- /dev/null +++ b/tests/LocalFailureReportTest.php @@ -0,0 +1,85 @@ +localActivity('local-failure', [], ['retry_policy' => ['max_attempts' => 1]]); + return 'unexpected-success'; + } catch (ActivityFailed $failure) { + $observations[] = [$failure->getMessage(), $failure->failureType, $failure->nonRetryable]; + return str_contains($failure->getMessage(), '[truncated failure report;') ? 'bounded-branch' : 'unbounded-branch'; + } + }; + $calls = 0; + $original = str_repeat('雪', 1024 * 1024); + $local = static function () use (&$calls, $original): array { + ++$calls; + return ['outcome' => 'failed', 'message' => $original, 'exception_type' => 'OriginalLocalFailure', + 'non_retryable' => true, 'attempts' => [['attempt_number' => 1, 'outcome' => 'failed', 'message' => $original, + 'exception_type' => 'OriginalLocalFailure', 'non_retryable' => true, 'duration_ms' => 0, 'heartbeats' => []]]]; + }; + $codec = new AvroPayloadCodec(); + $fresh = (new Replayer($codec))->replay($handler, [], [], 'queue', localActivityExecutor: $local); + self::assertNull($fresh->terminalFailure); + self::assertSame(['record_local_activity', 'complete_workflow'], array_column($fresh->commands, 'type')); + $recorded = $fresh->commands[0]; + self::assertLessThanOrEqual(63488, strlen($recorded['message'])); + self::assertSame(1, preg_match('//u', $recorded['message'])); + self::assertStringContainsString('3145728', $recorded['message']); + self::assertSame($recorded['message'], $recorded['attempts'][0]['message']); + self::assertSame($recorded['message'], $observations[0][0]); + $history = [ + ['event_type' => 'ActivityScheduled', 'payload' => ['sequence' => 1, 'activity_type' => 'local-failure', + 'execution_mode' => 'local', 'arguments' => $codec->envelope([])]], + ['event_type' => 'ActivityFailed', 'payload' => $recorded + ['sequence' => 1, 'execution_mode' => 'local']], + ]; + $cold = (new Replayer($codec))->replay($handler, $history, [], 'queue', localActivityExecutor: static function (): never { + self::fail('Cold replay repeated the local side effect.'); + }); + self::assertNull($cold->terminalFailure); + self::assertSame(['complete_workflow'], array_column($cold->commands, 'type')); + self::assertSame($observations[0], $observations[1]); + self::assertSame($fresh->commands[1]['result'], $cold->commands[0]['result']); + self::assertSame(1, $calls); + } + + public function testAggregateAttemptMessagesFitBeforeObservationWithoutChangingRetries(): void + { + $message = str_repeat("\0雪", 1000000); + $attempts = []; + for ($number = 1; $number <= 30; ++$number) { + $attempts[] = ['attempt_id' => 'original-attempt-'.$number, 'attempt_number' => $number, + 'outcome' => 'failed', 'message' => $message, 'exception_type' => 'OriginalFailure', + 'non_retryable' => false, 'retry_reason' => 'failure', 'backoff_seconds' => 3, 'heartbeats' => []]; + } + $outcome = ['outcome' => 'failed', 'message' => $message, 'exception_type' => 'OriginalFailure', + 'non_retryable' => false, 'attempts' => $attempts]; + $bounded = LocalFailureReports::bound($outcome, 128 * 1024); + self::assertLessThanOrEqual(128 * 1024, JsonDocument::size($bounded)); + self::assertCount(30, $bounded['attempts']); + foreach ($bounded['attempts'] as $index => $attempt) { + unset($attempt['message']); + $original = $attempts[$index]; + unset($original['message']); + self::assertSame($original, $attempt); + } + self::assertSame($bounded['message'], $bounded['attempts'][29]['message']); + self::assertSame($bounded, LocalFailureReports::bound($bounded, 128 * 1024)); + } +} From afbd2719ae0109d4cfdfa9ade7dbef8e1712df65 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:36:49 +0000 Subject: [PATCH 07/19] Declare local activity capability in failure-delivery fixture --- scripts/qualify-failure-report-delivery.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index f2bb9181..bce8baec 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -103,7 +103,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $worker->registerActivity('qualification.failure.local', static function (DurableWorkflow\Worker\ActivityContext $_context): never { throw new FailureDeliveryQualificationException(str_repeat('雪', 1024 * 1024)); }); -$client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes'], +$client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes', 'local_activities'], workflowCommandContracts: $worker->contracts()['workflow_commands']); $client->registerWorker($activityWorkerId, $queue, [], ['qualification.failure.activity']); $result = ['sdk_version' => Composer\InstalledVersions::getPrettyVersion('durable-workflow/sdk'), From f1904b6e2a0732de20420a9526aaa11c0936f54b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:39:05 +0000 Subject: [PATCH 08/19] Drive legitimate worker turns throughout failure-delivery qualification --- scripts/qualify-failure-report-delivery.php | 31 +++++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index bce8baec..4c37564c 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -22,6 +22,23 @@ function requireDelivery(bool $condition, string $message): void } } +/** Drive the ordinary worker while waiting for the selected run, with a finite fixture budget. */ +function finishFailureWorkflow(Client $control, Worker $worker, DurableWorkflow\WorkflowHandle $handle, ?Worker $activities = null): mixed +{ + $deadline = microtime(true) + 30; + do { + $execution = $control->describeWorkflow($handle->workflowId, $handle->selectedRunId); + if ($execution->isTerminal === true || in_array($execution->status, + ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { + return $handle->resultOfSelectedRun(1, 0.05); + } + $worker->tick(0); + $activities?->tick(0); + usleep(10000); + } while (microtime(true) < $deadline); + throw new FailureDeliveryQualificationException('The selected run did not finish within its fixture budget.'); +} + $http = new class implements Psr\Http\Client\ClientInterface { public array $reports = []; public array $claims = []; @@ -101,6 +118,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes }; $worker->registerWorkflow('qualification.failure', $workflowHandler); $worker->registerActivity('qualification.failure.local', static function (DurableWorkflow\Worker\ActivityContext $_context): never { + requireDelivery($_context->activityType === 'qualification.failure.local', 'Wrong local callback context.'); throw new FailureDeliveryQualificationException(str_repeat('雪', 1024 * 1024)); }); $client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes', 'local_activities'], @@ -135,7 +153,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $reporter->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], $message, $type, true, $details, $stack); $worker->tick(0); - $output = $handle->resultOfSelectedRun(15, 0.05); + $output = finishFailureWorkflow($control, $worker, $handle); requireDelivery($output['type'] === $type && $output['marked'] === ($kind !== 'small'), 'Original workflow failure mismatch.'); requireDelivery($output['non_retryable'] === true, 'The non-retryable decision changed.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); @@ -148,7 +166,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes } $handle = $control->startWorkflow('qualification.failure', $namespace.'-local', $queue, ['local-failure'], runTimeoutSeconds: 120); $worker->tick(0); -$output = $handle->resultOfSelectedRun(15, 0.05); +$output = finishFailureWorkflow($control, $worker, $handle); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; $failed = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityFailed')); @@ -178,6 +196,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes transport: new DurableWorkflow\Transport\Psr18Transport(client: new GuzzleHttp\Client(['handler' => $stack, 'http_errors' => false]))); $managedWorker = new Worker($managedClient, $queue, workerId: $managedId, enableCooperativeCancellation: true); $managedWorker->registerActivity('qualification.failure.managed', static function (DurableWorkflow\Worker\ActivityContext $_context): never { + requireDelivery($_context->activityType === 'qualification.failure.managed', 'Wrong isolated callback context.'); throw new FailureDeliveryQualificationException(str_repeat('m', 3 * 1024 * 1024)); }); $managedClient->registerWorker($managedId, $queue, [], ['qualification.failure.managed'], ['cooperative_cancellation']); @@ -185,7 +204,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $worker->tick(0); $managedWorker->tick(0); $worker->tick(0); -$output = $handle->resultOfSelectedRun(15, 0.05); +$output = finishFailureWorkflow($control, $worker, $handle, $managedWorker); requireDelivery($output['type'] === FailureDeliveryQualificationException::class && $output['marked'] === true, 'The original workflow did not receive its isolated callback failure.'); requireDelivery($output['non_retryable'] === false, 'The callback retry classification changed.'); @@ -216,7 +235,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes requireDelivery($retryAttempt !== $originalAttempt, 'The failed attempt was reused as publication authority.'); $client->completeActivityTask($retry['task_id'], $retryAttempt, $retry['lease_owner'], 42); $worker->tick(0); -$output = $handle->resultOfSelectedRun(15, 0.05); +$output = finishFailureWorkflow($control, $worker, $handle); requireDelivery($output === ['activity_value' => 42], 'The original workflow did not receive the legitimate retry result.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; @@ -233,7 +252,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'OriginalWorkflowTaskFailure', stackTrace: $large); requireDelivery(($ack['recorded'] ?? null) === true, 'Workflow-task failure not recorded.'); $worker->tick(0); -$output = $handle->resultOfSelectedRun(15, 0.05); +$output = finishFailureWorkflow($control, $worker, $handle); requireDelivery($output === ['fresh_turn' => true], 'The workflow did not receive its fresh legitimate turn.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; @@ -248,7 +267,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $handle = $control->startWorkflow('qualification.failure', $namespace.'-terminal', $queue, ['terminal'], runTimeoutSeconds: 120); $worker->tick(0); try { - $handle->resultOfSelectedRun(15, 0.05); + finishFailureWorkflow($control, $worker, $handle); throw new FailureDeliveryQualificationException('Terminal workflow unexpectedly succeeded.'); } catch (WorkflowFailed $failure) { requireDelivery($failure->failureType === FailureDeliveryQualificationException::class, 'Workflow failure type changed.'); From b85942136b5fb9d67b9bbeb5287bc900e8f7462d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:43:02 +0000 Subject: [PATCH 09/19] Distinguish transmitted callback traces from published history visibility --- scripts/qualify-failure-report-delivery.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index 4c37564c..78d6523b 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -68,6 +68,8 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'message_bytes' => strlen($failure['message']), 'marked' => str_contains($failure['message'], FAILURE_DELIVERY_MARKER), 'stack_bytes' => strlen($failure['stack_trace'] ?? $failure['exception']['stack_trace'] ?? ''), + 'stack_includes_qualifier' => str_contains($failure['stack_trace'] ?? $failure['exception']['stack_trace'] ?? '', 'qualify-failure-report-delivery.php'), + 'diagnostics' => $failure['diagnostics'] ?? $failure['exception']['diagnostics'] ?? null, 'message_sha256' => hash('sha256', $failure['message']), 'non_retryable' => $failure['non_retryable'] ?? null]; } if (str_ends_with($path, '/poll')) { @@ -211,11 +213,16 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; $failed = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityFailed')); -$failureWire = json_encode($failed, JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES); -requireDelivery(str_contains($failureWire, '3145728') && str_contains($failureWire, 'qualify-failure-report-delivery.php'), - 'Original callback trace or original-size diagnostics missing from history.'); +$failurePayload = $failed[0]['payload'] ?? []; +requireDelivery(($failurePayload['exception']['diagnostics']['truncation']['fields']['message']['original_bytes'] ?? null) === 3145728, + 'Original callback size diagnostics missing from history.'); +$report = $http->reports[array_key_last($http->reports)]; +requireDelivery($report['stack_bytes'] > 0 && $report['stack_includes_qualifier'] === true, + 'The SDK did not transmit the original isolated callback trace.'); $result['scenarios']['cooperative-worker'] = ['workflow_id' => $handle->workflowId, 'run_id' => $handle->selectedRunId, - 'output' => $output, 'original_callback_trace_and_sizes_in_history' => true]; + 'output' => $output, 'original_size_in_history' => true, 'transmitted_original_stack_bytes' => $report['stack_bytes'], + 'history_stack_present' => isset($failurePayload['exception']['stack_trace']), + 'history_stack_visibility' => 'Published runtime omits the transmitted stack from public history; shared225 runtime follow-through remains.']; $handle = $control->startWorkflow('qualification.failure', $namespace.'-retry', $queue, ['retry'], runTimeoutSeconds: 120); $worker->tick(0); $original = $client->pollActivityTask($activityWorkerId, $queue, 0); From 36e459e0a3f56486c8237114e7bbad41e8130f47 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:47:37 +0000 Subject: [PATCH 10/19] Maintain original fixture registrations during extended failure controls --- scripts/qualify-failure-report-delivery.php | 45 +++++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index 78d6523b..7782869c 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -23,7 +23,7 @@ function requireDelivery(bool $condition, string $message): void } /** Drive the ordinary worker while waiting for the selected run, with a finite fixture budget. */ -function finishFailureWorkflow(Client $control, Worker $worker, DurableWorkflow\WorkflowHandle $handle, ?Worker $activities = null): mixed +function finishFailureWorkflow(Client $control, Worker $worker, DurableWorkflow\WorkflowHandle $handle, Closure $maintain, ?Worker $activities = null): mixed { $deadline = microtime(true) + 30; do { @@ -32,6 +32,7 @@ function finishFailureWorkflow(Client $control, Worker $worker, DurableWorkflow\ ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { return $handle->resultOfSelectedRun(1, 0.05); } + $maintain(); $worker->tick(0); $activities?->tick(0); usleep(10000); @@ -126,13 +127,30 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $client->registerWorker($workerId, $queue, ['qualification.failure'], [], ['sticky_execution', 'typed_search_attributes', 'local_activities'], workflowCommandContracts: $worker->contracts()['workflow_commands']); $client->registerWorker($activityWorkerId, $queue, [], ['qualification.failure.activity']); +$registeredWorkers = [$workerId => $client, $activityWorkerId => $client]; +$lastHeartbeats = []; +$maintenance = []; +$maintain = static function () use (&$registeredWorkers, &$lastHeartbeats, &$maintenance): void { + foreach ($registeredWorkers as $identity => $connection) { + $now = hrtime(true) / 1e9; + if ($now - ($lastHeartbeats[$identity] ?? 0) < 5) { + continue; + } + // tick() deliberately leaves registration lifecycle to its caller. + $reply = $connection->heartbeatWorker($identity); + $lastHeartbeats[$identity] = $now; + $maintenance[] = ['worker_id' => $identity, 'registration_id' => $reply['registration_id'] ?? null, + 'registration_incarnation' => $reply['registration_incarnation'] ?? null]; + } +}; $result = ['sdk_version' => Composer\InstalledVersions::getPrettyVersion('durable-workflow/sdk'), 'sdk_reference' => Composer\InstalledVersions::getReference('durable-workflow/sdk'), 'server_version' => $admin->clusterInfo()->version, 'namespace' => $namespace, 'scenarios' => []]; -register_shutdown_function(static function () use (&$result, $http): void { +register_shutdown_function(static function () use (&$result, $http, &$maintenance): void { if (!isset($result['pass'])) { $result['reports'] = $http->reports; $result['claims'] = $http->claims; + $result['maintenance'] = $maintenance; $result['incomplete'] = true; $result['last_error'] = error_get_last(); echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; @@ -141,6 +159,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $large = str_repeat('m', 3 * 1024 * 1024); foreach (['message', 'stack', 'combined', 'escaped', 'details', 'lower-limit', 'small'] as $kind) { $handle = $control->startWorkflow('qualification.failure', $namespace.'-'.$kind, $queue, [$kind], runTimeoutSeconds: 120); + $maintain(); $worker->tick(0); $task = $client->pollActivityTask($activityWorkerId, $queue, 0); requireDelivery(is_array($task), 'The activity was not leased for '.$kind); @@ -154,8 +173,9 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $reporter = $kind === 'lower-limit' ? $client->withFailureReportLimit(4096) : $client; $reporter->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], $message, $type, true, $details, $stack); + $maintain(); $worker->tick(0); - $output = finishFailureWorkflow($control, $worker, $handle); + $output = finishFailureWorkflow($control, $worker, $handle, $maintain); requireDelivery($output['type'] === $type && $output['marked'] === ($kind !== 'small'), 'Original workflow failure mismatch.'); requireDelivery($output['non_retryable'] === true, 'The non-retryable decision changed.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); @@ -167,8 +187,9 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'lease_owner' => $task['lease_owner'], 'output' => $output, 'event_counts' => $counts]; } $handle = $control->startWorkflow('qualification.failure', $namespace.'-local', $queue, ['local-failure'], runTimeoutSeconds: 120); +$maintain(); $worker->tick(0); -$output = finishFailureWorkflow($control, $worker, $handle); +$output = finishFailureWorkflow($control, $worker, $handle, $maintain); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; $failed = array_values(array_filter($events, static fn (array $event): bool => $event['event_type'] === 'ActivityFailed')); @@ -202,11 +223,14 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes throw new FailureDeliveryQualificationException(str_repeat('m', 3 * 1024 * 1024)); }); $managedClient->registerWorker($managedId, $queue, [], ['qualification.failure.managed'], ['cooperative_cancellation']); +$registeredWorkers[$managedId] = $managedClient; $handle = $control->startWorkflow('qualification.failure', $namespace.'-cooperative-worker', $queue, ['cooperative-worker'], runTimeoutSeconds: 120); +$maintain(); $worker->tick(0); $managedWorker->tick(0); +$maintain(); $worker->tick(0); -$output = finishFailureWorkflow($control, $worker, $handle, $managedWorker); +$output = finishFailureWorkflow($control, $worker, $handle, $maintain, $managedWorker); requireDelivery($output['type'] === FailureDeliveryQualificationException::class && $output['marked'] === true, 'The original workflow did not receive its isolated callback failure.'); requireDelivery($output['non_retryable'] === false, 'The callback retry classification changed.'); @@ -224,6 +248,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'history_stack_present' => isset($failurePayload['exception']['stack_trace']), 'history_stack_visibility' => 'Published runtime omits the transmitted stack from public history; shared225 runtime follow-through remains.']; $handle = $control->startWorkflow('qualification.failure', $namespace.'-retry', $queue, ['retry'], runTimeoutSeconds: 120); +$maintain(); $worker->tick(0); $original = $client->pollActivityTask($activityWorkerId, $queue, 0); requireDelivery(is_array($original), 'Retry control original attempt was not leased.'); @@ -241,8 +266,9 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $retryAttempt = $retry['activity_attempt_id'] ?? $retry['attempt_id']; requireDelivery($retryAttempt !== $originalAttempt, 'The failed attempt was reused as publication authority.'); $client->completeActivityTask($retry['task_id'], $retryAttempt, $retry['lease_owner'], 42); +$maintain(); $worker->tick(0); -$output = finishFailureWorkflow($control, $worker, $handle); +$output = finishFailureWorkflow($control, $worker, $handle, $maintain); requireDelivery($output === ['activity_value' => 42], 'The original workflow did not receive the legitimate retry result.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; @@ -258,8 +284,9 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes $ack = $client->failWorkflowTask($task['task_id'], $task['lease_owner'], $task['workflow_task_attempt'], $large, 'OriginalWorkflowTaskFailure', stackTrace: $large); requireDelivery(($ack['recorded'] ?? null) === true, 'Workflow-task failure not recorded.'); +$maintain(); $worker->tick(0); -$output = finishFailureWorkflow($control, $worker, $handle); +$output = finishFailureWorkflow($control, $worker, $handle, $maintain); requireDelivery($output === ['fresh_turn' => true], 'The workflow did not receive its fresh legitimate turn.'); $history = $control->workflowHistory($handle->workflowId, $handle->selectedRunId, 100); $events = $history['events'] ?? $history['history_events'] ?? []; @@ -272,9 +299,10 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes 'ack' => $ack, 'history_failure_events' => count($failureEvents), 'fresh_turn_result' => $output, 'history_visibility' => 'Published runtime emits no WorkflowTaskFailed history event; parent225 runtime follow-through remains.']; $handle = $control->startWorkflow('qualification.failure', $namespace.'-terminal', $queue, ['terminal'], runTimeoutSeconds: 120); +$maintain(); $worker->tick(0); try { - finishFailureWorkflow($control, $worker, $handle); + finishFailureWorkflow($control, $worker, $handle, $maintain); throw new FailureDeliveryQualificationException('Terminal workflow unexpectedly succeeded.'); } catch (WorkflowFailed $failure) { requireDelivery($failure->failureType === FailureDeliveryQualificationException::class, 'Workflow failure type changed.'); @@ -287,5 +315,6 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes } $result['reports'] = $http->reports; $result['claims'] = $http->claims; +$result['maintenance'] = $maintenance; $result['pass'] = true; echo json_encode($result, JSON_THROW_ON_ERROR | JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)."\n"; From b080bd8ef7bb7c58c9fd3dc540e4be3e85b4fc1d Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:54:01 +0000 Subject: [PATCH 11/19] Keep normal payload budgets and plan uploads for mixed failure reports --- src/Client.php | 3 +- src/Failure/FailureReportLimiter.php | 77 ++++++++++++++++++++----- src/Transport/RuntimePayloadUploads.php | 5 +- tests/FailureReportTest.php | 13 +++++ tests/RuntimePayloadUploadTest.php | 39 +++++++++++++ 5 files changed, 121 insertions(+), 16 deletions(-) diff --git a/src/Client.php b/src/Client.php index 71fda799..dc74576a 100644 --- a/src/Client.php +++ b/src/Client.php @@ -2651,7 +2651,8 @@ private function request( $budget ??= $worker && $this->boundedWorkerRequests ? new RequestBudget($pollSeconds + 5) : null; if ($body !== null) { $body = $this->failureReports->request($path, $body, false); - $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget); + $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget, + $this->failureReports->limitForRequest($path, $body)); $body = $this->failureReports->request($path, $body); } $activityLease = $worker ? $this->activityLeaseBinding->current() : null; diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index 4beffa14..1bc0757c 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -22,25 +22,74 @@ public function __construct(private readonly FailureReportLimits $limits, privat * @return array */ public function request(string $path, array $body, bool $enforceRequestLimit = true): array + { + $kind = $this->reportKind($path, $body); + if ($kind === null) { + return $body; + } + if ($kind === 'prepared') { + return $this->preparedOutcome($path, $body); + } + $carrier = null; + if ($kind === 'failure') { + $carrier = str_contains($path, '/activity-tasks/') ? 'diagnostics' : 'stack_trace'; + } + return $this->bound($body, $carrier, $kind === 'commands', $this->effectiveLimit(), strlen($path), $enforceRequestLimit); + } + + /** @param array $body */ + public function limitForRequest(string $path, array $body): ?int + { + return $this->reportKind($path, $body) === null ? null : max(1, $this->effectiveLimit() - strlen($path)); + } + + /** @param array $body */ + private function reportKind(string $path, array $body): ?string { if (preg_match('~^/worker/workflow-tasks/[^/]+/local-activities/[^/]+/outcome$~D', $path) === 1 && is_array($body['report'] ?? null) && is_string($body['report']['message'] ?? null)) { - return $this->preparedOutcome($path, $body); + return 'prepared'; } - $endpoint = preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path, $matches) === 1 - && is_array($body['failure'] ?? null); - $commands = preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 - && is_array($body['commands'] ?? null) - && array_filter($body['commands'], static fn (mixed $command): bool => is_array($command) - && in_array($command['type'] ?? null, ['fail_workflow', 'fail_update', 'record_local_activity'], true)) !== []; - if (!$endpoint && !$commands) { - return $body; + if (preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path) === 1 + && is_array($body['failure'] ?? null)) { + return 'failure'; + } + if (preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 + && is_array($body['commands'] ?? null) && $this->hasFailureCommand($body['commands'])) { + return 'commands'; + } + return null; + } + + /** @param array $commands */ + private function hasFailureCommand(array $commands): bool + { + foreach ($commands as $command) { + if (is_array($command) && $this->isFailureCommand($command)) { + return true; + } + } + return false; + } + + /** @param array $command */ + private function isFailureCommand(array $command): bool + { + if (in_array($command['type'] ?? null, ['fail_workflow', 'fail_update'], true)) { + return true; + } + if (($command['type'] ?? null) !== 'record_local_activity') { + return false; + } + if (in_array($command['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true)) { + return true; + } + foreach ($command['attempts'] ?? [] as $attempt) { + if (is_array($attempt) && in_array($attempt['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true)) { + return true; + } } - $carrier = $endpoint ? match ($matches[1]) { - 'activity' => 'diagnostics', - default => 'stack_trace', - } : null; - return $this->bound($body, $carrier, $commands, $this->effectiveLimit(), strlen($path), $enforceRequestLimit); + return false; } /** @param array $body diff --git a/src/Transport/RuntimePayloadUploads.php b/src/Transport/RuntimePayloadUploads.php index 4e7b2728..88848d2a 100644 --- a/src/Transport/RuntimePayloadUploads.php +++ b/src/Transport/RuntimePayloadUploads.php @@ -40,7 +40,7 @@ public function __construct(private readonly Transport $transport, private reado * @param array $headers * @return array */ - public function request(array $body, string $method, string $path, bool $worker, array $headers, ?RequestBudget $budget = null): array + public function request(array $body, string $method, string $path, bool $worker, array $headers, ?RequestBudget $budget = null, ?int $requestBytesLimit = null): array { if (!$this->transport instanceof PayloadUploadTransport || !in_array(strtoupper($method), ['POST', 'PUT', 'PATCH'], true)) { return $body; @@ -103,6 +103,9 @@ public function request(array $body, string $method, string $path, bool $worker, throw $exception; } + if ($requestBytesLimit !== null) { + $policy['request_bytes'] = min($policy['request_bytes'], $requestBytesLimit); + } // Plan the entire request before uploading anything. Several individually // inline values can still exceed the ordinary JSON request limit. $selected = []; diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php index 311341ad..43bc7d77 100644 --- a/tests/FailureReportTest.php +++ b/tests/FailureReportTest.php @@ -196,6 +196,19 @@ public function testAggregateCompletionBudgetIncludesEveryCommandAndFixedBusines $limiter->request('/worker/workflow-tasks/task/complete', $body); } + public function testSuccessfulLocalRecordKeepsItsOrdinaryBusinessPayloadBudget(): void + { + $transport = new FakeTransport([[]]); + $client = new Client('http://recording.invalid', transport: $transport); + $result = $client->payloadCodec()->envelope(str_repeat('p', 900000)); + $command = ['type' => 'record_local_activity', 'activity_type' => 'business', 'outcome' => 'completed', + 'result' => $result, 'attempts' => [['attempt_number' => 1, 'outcome' => 'completed', 'heartbeats' => []]]]; + $client->completeWorkflowTask('task', 'owner', 1, [$command]); + self::assertCount(1, $transport->requests); + self::assertGreaterThan(1048576, JsonDocument::size($transport->requests[0]['body'])); + self::assertSame($result, $transport->requests[0]['body']['commands'][0]['result']); + } + public function testInvalidUtf8MessageRepairsBoundedPrefixWithoutChangingType(): void { $transport = new FakeTransport([[]]); diff --git a/tests/RuntimePayloadUploadTest.php b/tests/RuntimePayloadUploadTest.php index 2c00b0ce..37f37535 100644 --- a/tests/RuntimePayloadUploadTest.php +++ b/tests/RuntimePayloadUploadTest.php @@ -760,6 +760,45 @@ public function uploadPayload(string $uri, array $headers, string $blob, int $ti } } + public function testMixedLocalFailureUsesTheExistingUploadPlannerWithoutChangingBusinessPayloads(): void + { + foreach ([false, true] as $hasFailure) { + $policy = self::discovery(); + $policy['limits']['max_payload_bytes'] = 2097152; + $policy['namespace']['external_payload_storage']['threshold_bytes'] = 2097152; + $policy['namespace']['external_payload_storage']['transport']['limits']['max_payload_bytes'] = 8388608; + [$client, $http] = $this->client(discovery: $policy); + $value = ['text' => str_repeat('p', 900000), 'whole' => 1.0, 'zero' => -0.0]; + $envelope = $client->payloadCodec()->envelope($value); + $attempts = []; + if ($hasFailure) { + $attempts[] = ['attempt_number' => 1, 'outcome' => 'failed', 'message' => 'bounded failure', + 'exception_type' => 'OriginalFailure', 'non_retryable' => false, 'backoff_seconds' => 3, 'heartbeats' => []]; + } + $attempts[] = ['attempt_number' => count($attempts) + 1, 'outcome' => 'completed', 'heartbeats' => []]; + $client->completeWorkflowTask('original-task', 'original-owner', 7, [ + ['type' => 'record_local_activity', 'activity_type' => 'business', 'outcome' => 'completed', + 'result' => $envelope, 'attempts' => $attempts], + ]); + $completion = $http->requests[array_key_last($http->requests)]; + $body = $this->body($completion); + self::assertSame('original-owner', $body['lease_owner']); + self::assertSame(7, $body['workflow_task_attempt']); + self::assertSame($attempts, $body['commands'][0]['attempts']); + if ($hasFailure) { + self::assertCount(3, $http->requests); + self::assertSame($envelope['blob'], (string) $http->requests[1]->getBody()); + self::assertLessThanOrEqual(1048576, strlen((string) $completion->getBody())); + self::assertArrayHasKey('external_payload', $body['commands'][0]['result']); + self::assertSame(self::reference($envelope['blob']), $body['commands'][0]['result']['external_payload']); + } else { + self::assertCount(2, $http->requests); + self::assertGreaterThan(1048576, strlen((string) $completion->getBody())); + self::assertSame($envelope, $body['commands'][0]['result']); + } + } + } + private function client(?array $discovery = null, bool $workerOnly = false, ?Response $uploadResponse = null, bool $drainUnbound = false, ?Response $retryUploadResponse = null): array { $http = new class($discovery ?? self::discovery(), $uploadResponse, $drainUnbound, $retryUploadResponse) { From 54dfff6088c6d76f2b70231c7af761c9882aa624 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 15:56:31 +0000 Subject: [PATCH 12/19] Carry local size provenance separately from application error text --- src/Client.php | 5 +++-- src/Failure/LocalFailureReports.php | 34 ++++++----------------------- src/Worker.php | 10 +++++++-- tests/LocalFailureReportTest.php | 17 +++++++++++++++ 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/src/Client.php b/src/Client.php index dc74576a..c0114375 100644 --- a/src/Client.php +++ b/src/Client.php @@ -154,11 +154,12 @@ public function withFailureReportLimit(int $requestBytes): self /** @internal Freeze local diagnostic values before workflow code branches on them. * @param array $outcome + * @param array{message?: int, attempts?: array} $originalSizes * @return array */ - public function normalizeLocalFailure(array $outcome): array + public function normalizeLocalFailure(array $outcome, array $originalSizes = []): array { - return LocalFailureReports::bound($outcome, $this->failureReports->effectiveLimit()); + return LocalFailureReports::bound($outcome, $this->failureReports->effectiveLimit(), $originalSizes); } public function payloadCodec(): PayloadCodec diff --git a/src/Failure/LocalFailureReports.php b/src/Failure/LocalFailureReports.php index e7751486..7e45610e 100644 --- a/src/Failure/LocalFailureReports.php +++ b/src/Failure/LocalFailureReports.php @@ -13,16 +13,17 @@ final class LocalFailureReports private const DIAGNOSTICS = "\n[truncation diagnostics]\n"; /** @param array $outcome + * @param array{message?: int, attempts?: array} $originalSizes * @return array */ - public static function bound(array $outcome, int $requestLimit = FailureReportLimits::REQUEST_BYTES): array + public static function bound(array $outcome, int $requestLimit = FailureReportLimits::REQUEST_BYTES, array $originalSizes = []): array { $messageLimit = FailureReportLimits::MESSAGE_BYTES; for ($pass = 0; $pass < 18; ++$pass) { - $bounded = self::fragment($outcome, $messageLimit); + $bounded = self::fragment($outcome, $messageLimit, $originalSizes['message'] ?? null); foreach ($outcome['attempts'] ?? [] as $index => $attempt) { if (is_array($attempt)) { - $bounded['attempts'][$index] = self::fragment($attempt, $messageLimit); + $bounded['attempts'][$index] = self::fragment($attempt, $messageLimit, $originalSizes['attempts'][$index] ?? null); } } // A successful result keeps its independent business-payload policy. @@ -43,17 +44,15 @@ public static function bound(array $outcome, int $requestLimit = FailureReportLi /** @param array $fragment * @return array */ - private static function fragment(array $fragment, int $limit): array + private static function fragment(array $fragment, int $limit, ?int $sourceBytes = null): array { if (!is_string($fragment['message'] ?? null)) { return $fragment; } $message = trim($fragment['message']); $fragment['message'] = $message; - $originalBytes = self::originalBytes($message); - $existingDiagnostics = str_contains($message, self::DIAGNOSTICS); - if (strlen($message) <= $limit && preg_match('//u', $message) === 1 - && ($originalBytes === strlen($message) || $existingDiagnostics)) { + $originalBytes = max(strlen($message), $sourceBytes ?? 0); + if (strlen($message) <= $limit && preg_match('//u', $message) === 1 && $originalBytes === strlen($message)) { return $fragment; } $suffix = self::DIAGNOSTICS.JsonDocument::encode(['message' => ['original_bytes' => $originalBytes]]); @@ -63,23 +62,4 @@ private static function fragment(array $fragment, int $limit): array return $fragment; } - private static function originalBytes(string $message): int - { - $position = strrpos($message, self::DIAGNOSTICS); - if ($position !== false) { - try { - $value = json_decode(substr($message, $position + strlen(self::DIAGNOSTICS)), true, flags: JSON_THROW_ON_ERROR); - $bytes = is_array($value) ? ($value['message']['original_bytes'] ?? null) : null; - if (is_int($bytes) && $bytes >= 0) { - return $bytes; - } - } catch (\JsonException) { - // Caller text resembling a marker remains ordinary text. - } - } - if (preg_match('/\n\[truncated failure report; original message ([0-9]+) bytes\]$/D', $message, $matches) === 1) { - return (int) $matches[1]; - } - return strlen($message); - } } diff --git a/src/Worker.php b/src/Worker.php index 143d6ed0..8e597792 100644 --- a/src/Worker.php +++ b/src/Worker.php @@ -3047,6 +3047,7 @@ private function executeLocalActivity( $heartbeatTimeout = $options['heartbeat_timeout'] ?? null; $executionStartedAt = $this->now(); $attempts = []; + $localFailureSizes = []; $totalHeartbeatCount = 0; for ($attemptNumber = 1; $attemptNumber <= $maxAttempts; ++$attemptNumber) { @@ -3245,7 +3246,7 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi 'heartbeats' => $heartbeats, ]; - return $this->client->normalizeLocalFailure(['outcome' => 'completed', 'result' => $result, 'attempts' => $attempts]); + return $this->client->normalizeLocalFailure(['outcome' => 'completed', 'result' => $result, 'attempts' => $attempts], $localFailureSizes); } catch (Throwable $exception) { if ($exception instanceof WorkflowClaimAborted || $exception instanceof FailureReportRefused) { throw $exception; @@ -3303,6 +3304,10 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi if ($message === '') { $message = "Local activity failed with {$type}."; } + $sourceBytes = $exception instanceof ActivityExecutionFailure + ? ($exception->originalDiagnostics['truncation']['fields']['message']['original_bytes'] ?? null) : null; + $localFailureSizes['attempts'][$attemptNumber - 1] = is_int($sourceBytes) + ? max($sourceBytes, strlen($message)) : strlen($message); $attempt = [ 'attempt_id' => $attemptId, 'attempt_number' => $attemptNumber, @@ -3336,6 +3341,7 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi continue; } + $localFailureSizes['message'] = $localFailureSizes['attempts'][$attemptNumber - 1]; return $this->client->normalizeLocalFailure([ 'outcome' => $cancelled ? 'cancelled' : ($timedOut ? 'timed_out' : 'failed'), 'message' => $message, @@ -3343,7 +3349,7 @@ function (int $relay, int $callback) use ($task, $attemptId, $activityType): voi 'non_retryable' => $isNonRetryable, ...($timeoutKind === null ? [] : ['timeout_kind' => $timeoutKind]), 'attempts' => $attempts, - ]); + ], $localFailureSizes); } } diff --git a/tests/LocalFailureReportTest.php b/tests/LocalFailureReportTest.php index 3f6eea4d..4f3e6f46 100644 --- a/tests/LocalFailureReportTest.php +++ b/tests/LocalFailureReportTest.php @@ -59,6 +59,23 @@ public function testInitialAndColdWorkflowBranchesObserveTheSameNormalizedFailur self::assertSame(1, $calls); } + public function testCallerTextResemblingADiagnosticMarkerStaysOrdinarySmallText(): void + { + $message = "caller literal\n[truncated failure report; original message 0 bytes]"; + $outcome = ['outcome' => 'failed', 'message' => $message, 'exception_type' => 'OriginalFailure', 'non_retryable' => false]; + self::assertSame($outcome, LocalFailureReports::bound($outcome)); + $outcome['message'] .= "\n[truncation diagnostics]\n{\"message\":{\"original_bytes\":0}}"; + self::assertSame($outcome, LocalFailureReports::bound($outcome)); + } + + public function testOnlyExplicitCallbackProvenanceCarriesAnEarlierOriginalSize(): void + { + $outcome = ['outcome' => 'failed', 'message' => 'already bounded', 'exception_type' => 'OriginalFailure', 'non_retryable' => false]; + $bounded = LocalFailureReports::bound($outcome, originalSizes: ['message' => 3145728]); + self::assertStringContainsString('3145728', $bounded['message']); + self::assertSame($bounded, LocalFailureReports::bound($bounded)); + } + public function testAggregateAttemptMessagesFitBeforeObservationWithoutChangingRetries(): void { $message = str_repeat("\0雪", 1000000); From 95f530439e48b665adb09bda3ce694eb2638c70b Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:02:35 +0000 Subject: [PATCH 13/19] Bound failure metadata before planning business payload uploads --- src/Client.php | 2 +- src/Transport/RuntimePayloadUploads.php | 86 ++++++++++++++++++++----- tests/FailureReportTest.php | 21 ++++-- 3 files changed, 86 insertions(+), 23 deletions(-) diff --git a/src/Client.php b/src/Client.php index c0114375..b086be97 100644 --- a/src/Client.php +++ b/src/Client.php @@ -2653,7 +2653,7 @@ private function request( if ($body !== null) { $body = $this->failureReports->request($path, $body, false); $body = $this->payloadUploads->request($body, $method, $path, $worker, $headers, $budget, - $this->failureReports->limitForRequest($path, $body)); + $this->failureReports); $body = $this->failureReports->request($path, $body); } $activityLease = $worker ? $this->activityLeaseBinding->current() : null; diff --git a/src/Transport/RuntimePayloadUploads.php b/src/Transport/RuntimePayloadUploads.php index 88848d2a..a2219ca1 100644 --- a/src/Transport/RuntimePayloadUploads.php +++ b/src/Transport/RuntimePayloadUploads.php @@ -7,6 +7,7 @@ use DurableWorkflow\Exception\ExternalPayloadException; use DurableWorkflow\Exception\TransportException; use DurableWorkflow\Version; +use DurableWorkflow\Failure\FailureReportLimiter; /** Namespace-scoped discovery and request-local, content-addressed uploads. */ final class RuntimePayloadUploads @@ -40,26 +41,17 @@ public function __construct(private readonly Transport $transport, private reado * @param array $headers * @return array */ - public function request(array $body, string $method, string $path, bool $worker, array $headers, ?RequestBudget $budget = null, ?int $requestBytesLimit = null): array + public function request(array $body, string $method, string $path, bool $worker, array $headers, ?RequestBudget $budget = null, ?FailureReportLimiter $failureReports = null): array { if (!$this->transport instanceof PayloadUploadTransport || !in_array(strtoupper($method), ['POST', 'PUT', 'PATCH'], true)) { return $body; } $slots = $this->paths($body, explode('?', $path)[0], $worker); - $payloads = []; - foreach ($slots as $segments) { - $value = $this->get($body, $segments); - if (is_array($value) && array_key_exists('external_payload', $value)) { - if (count($value) !== 2 || ($value['codec'] ?? null) !== 'avro') { - throw new ExternalPayloadException('Invalid runtime payload envelope.', 422, 'external_payload_unsupported'); - } - RuntimePayloads::validateReference($value['external_payload']); - } - $blob = is_string($value) ? $value : (is_array($value) && count($value) === 2 - && ($value['codec'] ?? null) === 'avro' ? ($value['blob'] ?? null) : null); - if (is_string($blob)) { - $payloads[] = ['path' => $segments, 'blob' => $blob]; - } + $payloads = $this->payloads($body, $slots); + $requestBytesLimit = $failureReports?->limitForRequest($path, $body); + if ($requestBytesLimit !== null && $failureReports !== null) { + $body = $this->failureMetadata($body, $payloads, $path, $failureReports); + $payloads = $this->payloads($body, $slots); } if ($payloads === []) { return $body; @@ -411,6 +403,70 @@ private function placeholder(array &$body, array $payload): array return $reference; } + /** @param array $body + * @param list> $slots + * @return list, blob: string}> + */ + private function payloads(array $body, array $slots): array + { + $payloads = []; + foreach ($slots as $segments) { + $value = $this->get($body, $segments); + if (is_array($value) && array_key_exists('external_payload', $value)) { + if (count($value) !== 2 || ($value['codec'] ?? null) !== 'avro') { + throw new ExternalPayloadException('Invalid runtime payload envelope.', 422, 'external_payload_unsupported'); + } + RuntimePayloads::validateReference($value['external_payload']); + } + $blob = is_string($value) ? $value : (is_array($value) && count($value) === 2 + && ($value['codec'] ?? null) === 'avro' ? ($value['blob'] ?? null) : null); + if (is_string($blob)) { + $payloads[] = ['path' => $segments, 'blob' => $blob]; + } + } + return $payloads; + } + + /** Bound diagnostic text against complete metadata before planning ordinary uploads. + * @param array $body + * @param list, blob: string}> $payloads + * @return array + */ + private function failureMetadata(array $body, array $payloads, string $path, FailureReportLimiter $reports): array + { + $metadata = $body; + $large = []; + foreach ($payloads as $payload) { + if (strlen($payload['blob']) > 1024 && ($payload['path'][0] ?? null) !== 'failure') { + $large[] = $payload['path']; + $reference = ['schema' => RuntimePayloads::SCHEMA, 'reference_id' => 'ep_00000000000000000000000000', + 'codec' => 'avro', 'size_bytes' => strlen($payload['blob']), 'sha256' => hash('sha256', $payload['blob'])]; + $this->put($metadata, $payload['path'], ['codec' => 'avro', 'external_payload' => $reference]); + } + } + $metadata = $reports->request($path, $metadata); + foreach ($large as $segments) { + $value = $this->get($metadata, $segments); + if (is_array($value) && isset($value['external_payload'])) { + $this->put($metadata, $segments, $this->get($body, $segments)); + } + } + return $metadata; + } + + /** @param array $body + * @param list $segments + */ + private function put(array &$body, array $segments, mixed $value): void + { + $target = &$body; + $last = array_pop($segments); + foreach ($segments as $key) { + $target = &$target[$key]; + } + $target[$last] = $value; + } + /** @param array $body * @param list $segments * @param array $reference diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php index 43bc7d77..bbb5eb15 100644 --- a/tests/FailureReportTest.php +++ b/tests/FailureReportTest.php @@ -34,14 +34,21 @@ public function sendRequest(RequestInterface $request): ResponseInterface } }; $client = new Client('https://recording.invalid', transport: new Psr18Transport(client: $http)); + if (in_array($scenario, ['lower-limit', 'lower-details'], true)) { + $client = $client->withFailureReportLimit(4096); + } $large = str_repeat('m', 3 * 1024 * 1024); - $message = in_array($scenario, ['message', 'combined'], true) ? $large : 'original message'; - $stack = in_array($scenario, ['stack', 'combined'], true) ? $large : null; + $message = in_array($scenario, ['message', 'combined', 'lower-limit'], true) ? $large : 'original message'; + $stack = in_array($scenario, ['stack', 'combined', 'lower-limit'], true) ? $large : null; if ($scenario === 'escaped') { $message = str_repeat("\0\"\\雪🦀", 300000); $stack = str_repeat("\0\"\\雪🦀", 300000); } - $details = $scenario === 'details' ? ['nested' => ['cause' => $large]] : ['whole' => 1.0, 'zero' => -0.0]; + $details = match ($scenario) { + 'details' => ['nested' => ['cause' => $large]], + 'lower-details' => ['text' => str_repeat('d', 45000)], + default => ['whole' => 1.0, 'zero' => -0.0], + }; $client->failActivityTask('original-task', 'original-attempt', 'original-owner', $message, 'OriginalFailure', true, $details, $stack, ['authored_retry_delay' => 12.5]); $reports = array_values(array_filter($http->requests, @@ -49,7 +56,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface self::assertCount(1, $reports); $request = $reports[0]; $wire = (string) $request->getBody(); - self::assertLessThanOrEqual(1048576, strlen($wire)); + self::assertLessThanOrEqual(in_array($scenario, ['lower-limit', 'lower-details'], true) ? 4096 : 1048576, strlen($wire)); self::assertSame('/api/worker/activity-tasks/original-task/fail', $request->getUri()->getPath()); $body = json_decode($wire, true, flags: JSON_THROW_ON_ERROR); self::assertSame('original-attempt', $body['activity_attempt_id']); @@ -69,10 +76,10 @@ public function sendRequest(RequestInterface $request): ResponseInterface self::assertStringContainsString('[truncated failure report;', $failure['message']); self::assertNotEmpty($failure['diagnostics']['truncation']['fields']); } - if ($scenario === 'details') { + if (in_array($scenario, ['details', 'lower-details'], true)) { self::assertArrayNotHasKey('details', $failure); self::assertTrue($failure['diagnostics']['truncation']['fields']['details']['omitted']); - } else { + } elseif ($scenario !== 'lower-limit') { self::assertSame($client->payloadCodec()->envelope($details), $failure['details']); self::assertSame($details, $client->payloadCodec()->decodeEnvelope($failure['details'])); } @@ -80,7 +87,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface public static function largeReports(): iterable { - foreach (['message', 'stack', 'combined', 'escaped', 'details', 'small'] as $case) { + foreach (['message', 'stack', 'combined', 'escaped', 'details', 'lower-limit', 'lower-details', 'small'] as $case) { yield $case => [$case]; } } From 97fc0d796e13039fd5a0792141102074d8042200 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:03:21 +0000 Subject: [PATCH 14/19] Qualify lower-budget diagnostic omission through published backends --- docs/portal/build/failures-retries.md | 11 +++++++++++ scripts/qualify-failure-report-delivery.php | 10 +++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/portal/build/failures-retries.md b/docs/portal/build/failures-retries.md index 6dea5123..fdfbda73 100644 --- a/docs/portal/build/failures-retries.md +++ b/docs/portal/build/failures-retries.md @@ -146,6 +146,17 @@ escaping are included. If required identity or fixed request content cannot fit, reporting refusal requiring configuration or identity repair, rather than a new application failure or permission to publish under a different attempt. +Successful local records keep their normal business-payload budget. When a +completion includes both failure diagnostics and a large business result, the +existing runtime upload protocol preserves the complete Avro payload while +keeping the final failure request bounded. If fixed content cannot fit and the +namespace cannot store that payload, reporting refuses explicitly. No partial +business payload is published. + +Local callback failures normalize before workflow code branches on them. The +first execution, persisted local record and fresh replay use the same bounded +message, type and retry decision. + Activity failures use existing diagnostic fields. Workflow-task failures use the existing stack string for structured truncation information. The published Server qualification records task failures and grants a legitimate retry turn, diff --git a/scripts/qualify-failure-report-delivery.php b/scripts/qualify-failure-report-delivery.php index 7782869c..b492a2d9 100644 --- a/scripts/qualify-failure-report-delivery.php +++ b/scripts/qualify-failure-report-delivery.php @@ -157,7 +157,7 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes } }); $large = str_repeat('m', 3 * 1024 * 1024); -foreach (['message', 'stack', 'combined', 'escaped', 'details', 'lower-limit', 'small'] as $kind) { +foreach (['message', 'stack', 'combined', 'escaped', 'details', 'lower-limit', 'lower-details', 'small'] as $kind) { $handle = $control->startWorkflow('qualification.failure', $namespace.'-'.$kind, $queue, [$kind], runTimeoutSeconds: 120); $maintain(); $worker->tick(0); @@ -168,9 +168,13 @@ public function observe(Psr\Http\Message\RequestInterface $request, Psr\Http\Mes if ($kind === 'escaped') { $message = $stack = str_repeat("\0\"\\雪🦀", 300000); } - $details = $kind === 'details' ? ['nested' => ['cause' => $large]] : ['whole' => 1.0, 'zero' => -0.0]; + $details = match ($kind) { + 'details' => ['nested' => ['cause' => $large]], + 'lower-details' => ['text' => str_repeat('d', 45000)], + default => ['whole' => 1.0, 'zero' => -0.0], + }; $type = 'OriginalFailure.'.$kind; - $reporter = $kind === 'lower-limit' ? $client->withFailureReportLimit(4096) : $client; + $reporter = in_array($kind, ['lower-limit', 'lower-details'], true) ? $client->withFailureReportLimit(4096) : $client; $reporter->failActivityTask($task['task_id'], $task['activity_attempt_id'] ?? $task['attempt_id'], $task['lease_owner'], $message, $type, true, $details, $stack); $maintain(); From 3ad8dbd92b687a088c9cc646126bf74521bfe883 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:10:43 +0000 Subject: [PATCH 15/19] Preserve omission provenance and add an immutable local-failure corpus --- scripts/ci/run-replay-regression-fixture.php | 23 ++++++++++- src/Failure/FailureReportLimiter.php | 24 +++++++++-- tests/FailureReportTest.php | 2 + tests/Support/ReplayRegressionFixture.php | 27 +++++++++++-- ...-local-failure-before-workflow-branch.json | 40 +++++++++++++++++++ 5 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/replay-regressions/bounded-local-failure-before-workflow-branch.json diff --git a/scripts/ci/run-replay-regression-fixture.php b/scripts/ci/run-replay-regression-fixture.php index ebb7c387..7678edf5 100644 --- a/scripts/ci/run-replay-regression-fixture.php +++ b/scripts/ci/run-replay-regression-fixture.php @@ -377,6 +377,7 @@ public function run(WorkflowContext $context): array final class ReplayRegressionConsumer { private const FIXTURE_SCHEMA = 'durable-workflow.replay-regression/v1'; + private const BOUNDED_LOCAL_FAILURE = 'golden.bounded-local-failure'; public static function executeFile(string $path): void { @@ -564,6 +565,16 @@ static function (ActivityContext $_context) use (&$localActivityInvocations): ne }, ); } + if ($workflowType === self::BOUNDED_LOCAL_FAILURE) { + $worker->registerActivity(self::BOUNDED_LOCAL_FAILURE, + static function (ActivityContext $context) use (&$localActivityInvocations): never { + if ($context->activityType !== self::BOUNDED_LOCAL_FAILURE) { + throw new RuntimeException('Unexpected bounded-local activity context.'); + } + ++$localActivityInvocations; + throw new RuntimeException(str_repeat('雪', 1024 * 1024)); + }); + } if ($workflowType === 'golden.activity-completion-backend-loss') { $worker->registerActivity( 'golden.backend-recovery', @@ -581,7 +592,7 @@ static function (ActivityContext $_context) use (&$localActivityInvocations): st if ($workflowType === 'golden.workflow-lease-recovery') { $worker->tick(0); } - if ($localActivityInvocations !== ($workflowType === 'golden.activity-completion-backend-loss' ? 1 : 0)) { + if ($localActivityInvocations !== (in_array($workflowType, ['golden.activity-completion-backend-loss', self::BOUNDED_LOCAL_FAILURE], true) ? 1 : 0)) { throw new RuntimeException("{$identity} repeated a recorded local activity during cold replay."); } @@ -872,6 +883,16 @@ private static function workflow(string $workflowType): callable return ['key' => $selected->key]; }, + self::BOUNDED_LOCAL_FAILURE => static function (WorkflowContext $context): array { + try { + $context->localActivity('golden.bounded-local-failure', [], ['retry_policy' => ['max_attempts' => 1]]); + return ['unexpected_success' => true]; + } catch (ActivityFailed $failure) { + return ['marked' => str_contains($failure->getMessage(), '[truncated failure report;'), + 'message_bytes' => strlen($failure->getMessage()), 'failure_type' => $failure->failureType, + 'non_retryable' => $failure->nonRetryable]; + } + }, 'golden.local-activity-recovered' => static fn (WorkflowContext $context): mixed => $context->localActivity('golden.local-receipt'), 'golden.local-activity-terminal-failure' => static function (WorkflowContext $context): never { diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index 1bc0757c..c635b886 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -245,14 +245,32 @@ private function priorChanges(array $original): array $result = []; foreach (['message', 'stack_trace', 'exception.message', ...self::METADATA] as $field) { $entry = $fields[$field] ?? null; - $bytes = is_array($entry) ? ($entry['original_bytes'] ?? null) : null; - if (is_int($bytes) && $bytes >= 0) { - $result[$field] = ['original_bytes' => $bytes]; + $retained = is_array($entry) ? $this->priorField($entry) : []; + if ($retained !== []) { + $result[$field] = $retained; } } return $result; } + /** @param array $entry + * @return array + */ + private function priorField(array $entry): array + { + $result = []; + $bytes = $entry['original_bytes'] ?? null; + if (is_int($bytes) && $bytes >= 0) { + $result['original_bytes'] = $bytes; + } + if (($entry['omitted'] ?? null) === true) { + $result['omitted'] = true; + $serialized = $entry['original_serialized_bytes'] ?? null; + $result['original_serialized_bytes'] = is_int($serialized) && $serialized >= 0 ? $serialized : null; + } + return $result; + } + /** @param array $report * @param array $metadata * @param array $changes diff --git a/tests/FailureReportTest.php b/tests/FailureReportTest.php index bbb5eb15..2ff37094 100644 --- a/tests/FailureReportTest.php +++ b/tests/FailureReportTest.php @@ -147,6 +147,8 @@ public function testLowerConfiguredLimitCountsEscapingAfterAllMarkers(): void self::assertFalse($body['failure']['non_retryable']); self::assertStringContainsString('[truncated failure report;', $body['failure']['message']); self::assertArrayNotHasKey('details', $body['failure']); + self::assertSame(JsonDocument::size($client->payloadCodec()->envelope(['oversized' => str_repeat('d', 100000)])), + $body['failure']['diagnostics']['truncation']['fields']['details']['original_serialized_bytes'] ?? null); } public function testSmallTransportBudgetRetainsMarkersInsteadOfAnArbitraryContentFloor(): void diff --git a/tests/Support/ReplayRegressionFixture.php b/tests/Support/ReplayRegressionFixture.php index d72c29cf..77ac55c3 100644 --- a/tests/Support/ReplayRegressionFixture.php +++ b/tests/Support/ReplayRegressionFixture.php @@ -25,6 +25,7 @@ final class ReplayRegressionFixture { private const FIXTURE_SCHEMA = 'durable-workflow.replay-regression/v1'; + private const BOUNDED_LOCAL_FAILURE = 'golden.bounded-local-failure'; /** @return list> */ public static function executeFile(string $path): array @@ -108,11 +109,17 @@ public static function execute(array $fixture): array } $localActivityInvocations = 0; $localActivityExecutor = in_array($workflowType, [ - 'golden.local-activity-terminal-failure', 'golden.local-activity-recovered', 'golden.prepared-local-cleanup', + self::BOUNDED_LOCAL_FAILURE, 'golden.local-activity-terminal-failure', 'golden.local-activity-recovered', 'golden.prepared-local-cleanup', 'golden.prepared-local-group', 'golden.prepared-local-scoped-cleanup', ], true) - ? static function () use (&$localActivityInvocations): array { + ? static function () use (&$localActivityInvocations, $workflowType): array { ++$localActivityInvocations; + if ($workflowType === self::BOUNDED_LOCAL_FAILURE) { + $message = str_repeat('雪', 1024 * 1024); + return ['outcome' => 'failed', 'message' => $message, 'exception_type' => RuntimeException::class, + 'non_retryable' => false, 'attempts' => [['attempt_number' => 1, 'outcome' => 'failed', + 'message' => $message, 'exception_type' => RuntimeException::class, 'non_retryable' => false, 'heartbeats' => []]]]; + } return [ 'outcome' => 'completed', @@ -194,7 +201,11 @@ public static function execute(array $fixture): array 'exception_type' => $exception::class, ]]; } - if ($localActivityInvocations !== 0) { + if ($workflowType === self::BOUNDED_LOCAL_FAILURE) { + if ($localActivityInvocations !== 1) { + throw new RuntimeException("{$identity} must execute its fresh local failure exactly once."); + } + } elseif ($localActivityInvocations !== 0) { throw new RuntimeException("{$identity} repeated a recorded local activity during cold replay."); } $declaredCommands = $fixture['command_sequence'] ?? null; @@ -583,6 +594,16 @@ private static function workflow(string $workflowType): callable return ['key' => $selected->key]; }, + self::BOUNDED_LOCAL_FAILURE => static function (WorkflowContext $context): array { + try { + $context->localActivity('golden.bounded-local-failure', [], ['retry_policy' => ['max_attempts' => 1]]); + return ['unexpected_success' => true]; + } catch (ActivityFailed $failure) { + return ['marked' => str_contains($failure->getMessage(), '[truncated failure report;'), + 'message_bytes' => strlen($failure->getMessage()), 'failure_type' => $failure->failureType, + 'non_retryable' => $failure->nonRetryable]; + } + }, 'golden.local-activity-recovered' => static fn (WorkflowContext $context): mixed => $context->localActivity('golden.local-receipt'), 'golden.local-activity-terminal-failure' => static function (WorkflowContext $context): never { diff --git a/tests/fixtures/replay-regressions/bounded-local-failure-before-workflow-branch.json b/tests/fixtures/replay-regressions/bounded-local-failure-before-workflow-branch.json new file mode 100644 index 00000000..f0b17e18 --- /dev/null +++ b/tests/fixtures/replay-regressions/bounded-local-failure-before-workflow-branch.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://raw.githubusercontent.com/durable-workflow/.github/main/regression-corpus/evidence-schema.json", + "fixture_schema": "durable-workflow.replay-regression/v1", + "id": "php-bounded-local-failure-before-workflow-branch", + "protocol_version": "1.18", + "bindings": [ + "php" + ], + "workflow": { + "type": "golden.bounded-local-failure", + "input": [] + }, + "history": [ + { + "event_type": "WorkflowStarted", + "payload": { + "workflow_type": "golden.bounded-local-failure" + } + } + ], + "expected": { + "command_sequence": [ + { + "type": "record_local_activity", + "outcome": "failed", + "exception_type": "RuntimeException", + "non_retryable": false + }, + { + "type": "complete_workflow", + "result": { + "marked": true, + "message_bytes": 63486, + "failure_type": "RuntimeException", + "non_retryable": false + } + } + ] + } +} From f35457ffce7f925c81b35924d268f6e3ea0f0e43 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:16:05 +0000 Subject: [PATCH 16/19] Remove the redundant nullable planner guard --- src/Transport/RuntimePayloadUploads.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Transport/RuntimePayloadUploads.php b/src/Transport/RuntimePayloadUploads.php index a2219ca1..1a4d1ef3 100644 --- a/src/Transport/RuntimePayloadUploads.php +++ b/src/Transport/RuntimePayloadUploads.php @@ -49,7 +49,7 @@ public function request(array $body, string $method, string $path, bool $worker, $slots = $this->paths($body, explode('?', $path)[0], $worker); $payloads = $this->payloads($body, $slots); $requestBytesLimit = $failureReports?->limitForRequest($path, $body); - if ($requestBytesLimit !== null && $failureReports !== null) { + if ($requestBytesLimit !== null) { $body = $this->failureMetadata($body, $payloads, $path, $failureReports); $payloads = $this->payloads($body, $slots); } From 88dcd07c400c251d21aa5ded4ea43074fb251068 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:17:46 +0000 Subject: [PATCH 17/19] Prepare compatible SDK 3.5.0 failure-report release --- CHANGELOG.md | 23 +++++++++++++++++++++++ composer.json | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50d4a170..aa9e7b12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,29 @@ project follows [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [3.5.0] - 2026-10-23 + +### Added + +- `Client::withFailureReportLimit()` narrows the complete encoded failure-report + budget. Transports may expose a lower known limit. Activity and workflow-task + reporting accept optional stack traces, and activity reporting accepts + diagnostics, after all existing arguments. + +### Fixed + +- Bound failure messages to 62 KiB, stacks to 128 KiB, combined details and + diagnostics to 64 KiB, and complete encoded reports to 1 MiB by default. + Markers and JSON escaping count toward these limits. Oversized details remain + whole valid Avro values or are omitted with original-size diagnostics. +- Preserve original failure types, retry decisions, authored delays and claim + identities. Local durable failures normalize before workflow code observes + them, so initial execution, persistence and cold replay agree. Cooperative + failure IPC has its own diagnostic budget, separate from business payloads. +- Keep ordinary successful payload budgets unchanged. Mixed failure/result + completions use the existing upload planner without changing or slicing the + business payload. Unfit required report content refuses explicitly. + ## [3.4.1] - 2026-10-23 ### Fixed diff --git a/composer.json b/composer.json index ab2d0803..1640f09e 100644 --- a/composer.json +++ b/composer.json @@ -93,7 +93,7 @@ } }, "durable-workflow": { - "product-train": "3.4.1", + "product-train": "3.5.0", "supported-server-versions": "2.5.0", "worker-protocol-version": "1.20", "control-plane-version": "2", From 4efb8d6b28fcd189f9260ec1042fc68f34ca8b93 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:29:21 +0000 Subject: [PATCH 18/19] Extract bounded upload stages and simplify failure classifiers --- scripts/ci/run-replay-regression-fixture.php | 65 +++-- src/Failure/FailureReportLimiter.php | 35 +-- src/Transport/RuntimePayloadUploads.php | 272 ++++++++++++------- 3 files changed, 224 insertions(+), 148 deletions(-) diff --git a/scripts/ci/run-replay-regression-fixture.php b/scripts/ci/run-replay-regression-fixture.php index 7678edf5..1663c4b0 100644 --- a/scripts/ci/run-replay-regression-fixture.php +++ b/scripts/ci/run-replay-regression-fixture.php @@ -555,36 +555,7 @@ private static function executeLayout( ); } $localActivityInvocations = 0; - if (in_array($workflowType, ['golden.local-activity-terminal-failure', 'golden.local-activity-recovered'], true)) { - $worker->registerActivity( - $workflowType === 'golden.local-activity-recovered' ? 'golden.local-receipt' : 'golden.local-failure', - static function (ActivityContext $_context) use (&$localActivityInvocations): never { - ++$localActivityInvocations; - - throw new RuntimeException('Cold replay repeated the recorded local activity side effect.'); - }, - ); - } - if ($workflowType === self::BOUNDED_LOCAL_FAILURE) { - $worker->registerActivity(self::BOUNDED_LOCAL_FAILURE, - static function (ActivityContext $context) use (&$localActivityInvocations): never { - if ($context->activityType !== self::BOUNDED_LOCAL_FAILURE) { - throw new RuntimeException('Unexpected bounded-local activity context.'); - } - ++$localActivityInvocations; - throw new RuntimeException(str_repeat('雪', 1024 * 1024)); - }); - } - if ($workflowType === 'golden.activity-completion-backend-loss') { - $worker->registerActivity( - 'golden.backend-recovery', - static function (ActivityContext $_context) use (&$localActivityInvocations): string { - ++$localActivityInvocations; - - return 'activity-after-backend-recovery'; - }, - ); - } + self::registerLocalActivities($worker, $workflowType, $localActivityInvocations); foreach ($tasks as $_task) { $worker->tick(0); gc_collect_cycles(); @@ -651,6 +622,40 @@ private static function assertExpected(string $identity, array $fixture, array $ self::assertMatches($expected, $observed, "{$identity}.expected"); } + private static function registerLocalActivities(Worker $worker, string $workflowType, int &$localActivityInvocations): void + { + if (in_array($workflowType, ['golden.local-activity-terminal-failure', 'golden.local-activity-recovered'], true)) { + $worker->registerActivity( + $workflowType === 'golden.local-activity-recovered' ? 'golden.local-receipt' : 'golden.local-failure', + static function (ActivityContext $_context) use (&$localActivityInvocations): never { + ++$localActivityInvocations; + + throw new RuntimeException('Cold replay repeated the recorded local activity side effect.'); + }, + ); + } + if ($workflowType === self::BOUNDED_LOCAL_FAILURE) { + $worker->registerActivity(self::BOUNDED_LOCAL_FAILURE, + static function (ActivityContext $context) use (&$localActivityInvocations): never { + if ($context->activityType !== self::BOUNDED_LOCAL_FAILURE) { + throw new UnexpectedValueException('Unexpected bounded-local activity context.'); + } + ++$localActivityInvocations; + throw new RuntimeException(str_repeat('雪', 1024 * 1024)); + }); + } + if ($workflowType === 'golden.activity-completion-backend-loss') { + $worker->registerActivity( + 'golden.backend-recovery', + static function (ActivityContext $_context) use (&$localActivityInvocations): string { + ++$localActivityInvocations; + + return 'activity-after-backend-recovery'; + }, + ); + } + } + /** @return callable(WorkflowContext, mixed ...$input): mixed */ private static function workflow(string $workflowType): callable { diff --git a/src/Failure/FailureReportLimiter.php b/src/Failure/FailureReportLimiter.php index c635b886..b7d057e2 100644 --- a/src/Failure/FailureReportLimiter.php +++ b/src/Failure/FailureReportLimiter.php @@ -46,19 +46,15 @@ public function limitForRequest(string $path, array $body): ?int /** @param array $body */ private function reportKind(string $path, array $body): ?string { - if (preg_match('~^/worker/workflow-tasks/[^/]+/local-activities/[^/]+/outcome$~D', $path) === 1 - && is_array($body['report'] ?? null) && is_string($body['report']['message'] ?? null)) { - return 'prepared'; - } - if (preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path) === 1 - && is_array($body['failure'] ?? null)) { - return 'failure'; - } - if (preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 - && is_array($body['commands'] ?? null) && $this->hasFailureCommand($body['commands'])) { - return 'commands'; - } - return null; + return match (true) { + preg_match('~^/worker/workflow-tasks/[^/]+/local-activities/[^/]+/outcome$~D', $path) === 1 + && is_array($body['report'] ?? null) && is_string($body['report']['message'] ?? null) => 'prepared', + preg_match('~^/worker/(activity|workflow|query)-tasks/[^/]+/fail$~D', $path) === 1 + && is_array($body['failure'] ?? null) => 'failure', + preg_match('~^/worker/workflow-tasks/[^/]+/complete$~D', $path) === 1 + && is_array($body['commands'] ?? null) && $this->hasFailureCommand($body['commands']) => 'commands', + default => null, + }; } /** @param array $commands */ @@ -81,15 +77,10 @@ private function isFailureCommand(array $command): bool if (($command['type'] ?? null) !== 'record_local_activity') { return false; } - if (in_array($command['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true)) { - return true; - } - foreach ($command['attempts'] ?? [] as $attempt) { - if (is_array($attempt) && in_array($attempt['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true)) { - return true; - } - } - return false; + $attempts = is_array($command['attempts'] ?? null) ? $command['attempts'] : []; + return in_array($command['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true) + || array_filter($attempts, static fn (mixed $attempt): bool => is_array($attempt) + && in_array($attempt['outcome'] ?? null, ['failed', 'timed_out', 'cancelled'], true)) !== []; } /** @param array $body diff --git a/src/Transport/RuntimePayloadUploads.php b/src/Transport/RuntimePayloadUploads.php index 1a4d1ef3..139280b9 100644 --- a/src/Transport/RuntimePayloadUploads.php +++ b/src/Transport/RuntimePayloadUploads.php @@ -9,7 +9,11 @@ use DurableWorkflow\Version; use DurableWorkflow\Failure\FailureReportLimiter; -/** Namespace-scoped discovery and request-local, content-addressed uploads. */ +/** Namespace-scoped discovery and request-local, content-addressed uploads. + * @phpstan-type PayloadSlot array{path: list, blob: string} + * @phpstan-type UploadPolicy array{threshold_bytes: int, max_bytes: int, request_bytes: int, timeout_seconds: int, status: string, completion_context?: bool, prepared_completion_context?: bool, scope_completion_context?: bool} + * @phpstan-type UploadRequest array{headers: array, path: string, worker: bool, budget: RequestBudget|null} + */ final class RuntimePayloadUploads { private const COMPLETION_SCHEMA = 'durable-workflow.v2.payload-completion-context.v1'; @@ -57,8 +61,23 @@ public function request(array $body, string $method, string $path, bool $worker, return $body; } + $policy = $this->requestPolicy($body, $path, $worker, $headers, $budget); + if ($requestBytesLimit !== null) { + $policy['request_bytes'] = min($policy['request_bytes'], $requestBytesLimit); + } + $selected = $this->plan($body, $payloads, $policy); + return $this->uploadSelected($body, $payloads, $selected, $policy, + ['headers' => $headers, 'path' => $path, 'worker' => $worker, 'budget' => $budget]); + } + + /** @param array $body + * @param array $headers + * @return UploadPolicy + */ + private function requestPolicy(array $body, string $path, bool $worker, array $headers, ?RequestBudget $budget): array + { try { - $policy = $this->policy($headers, $worker, $budget); + return $this->policy($headers, $worker, $budget); } catch (TransportException $exception) { $discovery = $exception->response; if ($worker @@ -95,9 +114,15 @@ public function request(array $body, string $method, string $path, bool $worker, throw $exception; } - if ($requestBytesLimit !== null) { - $policy['request_bytes'] = min($policy['request_bytes'], $requestBytesLimit); - } + } + + /** @param array $body + * @param list $payloads + * @param UploadPolicy $policy + * @return array> + */ + private function plan(array &$body, array &$payloads, array $policy): array + { // Plan the entire request before uploading anything. Several individually // inline values can still exceed the ordinary JSON request limit. $selected = []; @@ -125,98 +150,144 @@ public function request(array $body, string $method, string $path, bool $worker, throw new ExternalPayloadException('Namespace runtime external payload storage is not available.', 503, 'external_payload_unavailable'); } - $uploaded = []; - $upload = function (array $uploadHeaders, string $blob) use ($policy, $budget): array { - $uri = $this->baseUri.'/api/external-payloads/v1'; - if ($budget !== null && !$this->transport instanceof BoundedPayloadUploadTransport) { - throw new ExternalPayloadException('The cooperative transport must bound payload uploads.', 415, 'external_payload_unsupported'); - } - $deadline = hrtime(true) / 1e9 + $policy['timeout_seconds']; - $timeout = $policy['timeout_seconds']; - for ($attempt = 0; ; ++$attempt) { - try { - if ($budget === null) { - return $this->transport->uploadPayload($uri, $uploadHeaders, $blob, $timeout); - } - $response = $this->transport->uploadPayloadBounded($uri, $uploadHeaders, $blob, min($budget->remainingSeconds(), $timeout)); - $budget->remainingSeconds(); - - return $response; - } catch (TransportException $exception) { - // A reservation or object can already exist after this refusal. - // Retry only the upload, with identical bytes and completion - // context, once within the original upload/request budget. - $timeout = (int) floor($deadline - hrtime(true) / 1e9); - if ($attempt !== 0 || $exception->status !== 503 - || ($exception->response['schema'] ?? null) !== 'durable-workflow.v2.runtime-external-payload-error.v1' - || ($exception->response['reason'] ?? null) !== 'external_payload_unavailable' - || ($exception->response['retryable'] ?? null) !== true - || $timeout < 1) { - throw $exception; - } - $budget?->remainingSeconds(); + return $selected; + } + + /** @param array $uploadHeaders + * @param UploadPolicy $policy + * @return array + */ + private function uploadBlob(array $uploadHeaders, string $blob, array $policy, ?RequestBudget $budget): array + { + $transport = $this->transport; + if (!$transport instanceof PayloadUploadTransport) { + throw new ExternalPayloadException('The transport must support payload uploads.', 415, 'external_payload_unsupported'); + } + $uri = $this->baseUri.'/api/external-payloads/v1'; + if ($budget !== null && !$transport instanceof BoundedPayloadUploadTransport) { + throw new ExternalPayloadException('The cooperative transport must bound payload uploads.', 415, 'external_payload_unsupported'); + } + $deadline = hrtime(true) / 1e9 + $policy['timeout_seconds']; + $timeout = $policy['timeout_seconds']; + for ($attempt = 0; ; ++$attempt) { + try { + if ($budget === null) { + return $transport->uploadPayload($uri, $uploadHeaders, $blob, $timeout); } + $response = $transport->uploadPayloadBounded($uri, $uploadHeaders, $blob, min($budget->remainingSeconds(), $timeout)); + $budget->remainingSeconds(); + + return $response; + } catch (TransportException $exception) { + // A reservation or object can already exist after this refusal. + // Retry only the upload, with identical bytes and completion + // context, once within the original upload/request budget. + $timeout = (int) floor($deadline - hrtime(true) / 1e9); + if ($attempt !== 0 || $exception->status !== 503 + || ($exception->response['schema'] ?? null) !== 'durable-workflow.v2.runtime-external-payload-error.v1' + || ($exception->response['reason'] ?? null) !== 'external_payload_unavailable' + || ($exception->response['retryable'] ?? null) !== true + || $timeout < 1) { + throw $exception; + } + $budget?->remainingSeconds(); } - }; + } + } + + /** @param array $body + * @param list $payloads + * @param array> $selected + * @param UploadPolicy $policy + * @param UploadRequest $request + * @return array + */ + private function uploadSelected(array $body, array $payloads, array $selected, array $policy, array $request): array + { + $uploaded = []; foreach ($selected as $index => $expected) { - $blob = $payloads[$index]['blob']; $key = $expected['sha256'].':'.$expected['size_bytes']; - if (!isset($uploaded[$key])) { - try { - $uploadHeaders = array_replace($headers, [ - 'Content-Type' => 'application/octet-stream', - 'Accept' => 'application/json', - 'X-Durable-Workflow-Payload-Codec' => 'avro', - 'X-Durable-Workflow-Payload-Size' => (string) strlen($blob), - 'X-Durable-Workflow-Payload-SHA256' => $expected['sha256'], - ]); - try { - $response = $upload($uploadHeaders, $blob); - } catch (TransportException $refusal) { - $context = $worker && ($policy['completion_context'] ?? false) - ? $this->completionContext($body, $path, $payloads[$index]['path'], - ($policy['prepared_completion_context'] ?? false) - && ($headers['X-Durable-Workflow-Protocol-Version'] ?? null) === Version::COOPERATIVE_CANCELLATION_MINIMUM_WORKER_PROTOCOL, - ($policy['scope_completion_context'] ?? false) - && ($headers['X-Durable-Workflow-Protocol-Version'] ?? null) === Version::COOPERATIVE_CANCELLATION_MINIMUM_WORKER_PROTOCOL) : null; - if ($context === null || $refusal->status !== 503 - || ($refusal->response['reason'] ?? null) !== 'storage_pressure' - || ($refusal->response['storage_state'] ?? null) !== 'draining' - || ($refusal->response['request_admitted'] ?? null) !== false) { - throw $refusal; - } - // One capability-negotiated retry; never turn other pressure - // errors into retries or repeat application activity code. - $uploadHeaders[self::COMPLETION_HEADER] = $context; - $response = $upload($uploadHeaders, $blob); - } - } catch (TransportException $exception) { - $reason = $exception->response['reason'] ?? null; - $reason = is_string($reason) ? $reason : match ($exception->status) { - 401, 403 => 'external_payload_unauthorized', - 413 => 'external_payload_oversized', - 415, 422 => 'external_payload_unsupported', - default => 'external_payload_unavailable', - }; - throw new ExternalPayloadException('Runtime external payload upload failed.', $exception->status ?? 0, - $reason, $exception->response, $exception); - } - if (($response['schema'] ?? null) !== 'durable-workflow.v2.runtime-external-payload-upload.v1' - || ($response['transport_version'] ?? null) !== 1) { - throw new ExternalPayloadException('Unsupported runtime upload response.', 422, 'external_payload_unsupported'); - } - $reference = RuntimePayloads::validateReference($response['reference'] ?? null); - if ($reference['size_bytes'] !== strlen($blob) || !hash_equals($expected['sha256'], $reference['sha256'])) { - throw new ExternalPayloadException('Runtime upload reference differs from the submitted bytes.', 422, 'external_payload_integrity_mismatch'); - } - $uploaded[$key] = $reference; - } + $uploaded[$key] ??= $this->uploadReference($body, $payloads[$index], $expected, $policy, $request); $this->replace($body, $payloads[$index]['path'], $uploaded[$key]); } - return $body; } + /** @param array $body + * @param PayloadSlot $payload + * @param array $expected + * @param UploadPolicy $policy + * @param UploadRequest $request + * @return array + */ + private function uploadReference(array $body, array $payload, array $expected, array $policy, array $request): array + { + $blob = $payload['blob']; + $headers = array_replace($request['headers'], [ + 'Content-Type' => 'application/octet-stream', 'Accept' => 'application/json', + 'X-Durable-Workflow-Payload-Codec' => 'avro', 'X-Durable-Workflow-Payload-Size' => (string) strlen($blob), + 'X-Durable-Workflow-Payload-SHA256' => $expected['sha256'], + ]); + try { + $response = $this->sendWithCompletionFallback($body, $payload, $headers, $policy, $request); + } catch (TransportException $exception) { + $reason = $exception->response['reason'] ?? null; + $reason = is_string($reason) ? $reason : match ($exception->status) { + 401, 403 => 'external_payload_unauthorized', 413 => 'external_payload_oversized', + 415, 422 => 'external_payload_unsupported', default => 'external_payload_unavailable', + }; + throw new ExternalPayloadException('Runtime external payload upload failed.', $exception->status ?? 0, + $reason, $exception->response, $exception); + } + return $this->validateUploadReference($response, $blob, $expected); + } + + /** @param array $body + * @param PayloadSlot $payload + * @param array $headers + * @param UploadPolicy $policy + * @param UploadRequest $request + * @return array + */ + private function sendWithCompletionFallback(array $body, array $payload, array $headers, array $policy, array $request): array + { + try { + return $this->uploadBlob($headers, $payload['blob'], $policy, $request['budget']); + } catch (TransportException $refusal) { + $cooperative = ($headers['X-Durable-Workflow-Protocol-Version'] ?? null) === Version::COOPERATIVE_CANCELLATION_MINIMUM_WORKER_PROTOCOL; + $context = $request['worker'] && ($policy['completion_context'] ?? false) + ? $this->completionContext($body, $request['path'], $payload['path'], + ($policy['prepared_completion_context'] ?? false) && $cooperative, + ($policy['scope_completion_context'] ?? false) && $cooperative) : null; + if ($context === null || $refusal->status !== 503 + || ($refusal->response['reason'] ?? null) !== 'storage_pressure' + || ($refusal->response['storage_state'] ?? null) !== 'draining' + || ($refusal->response['request_admitted'] ?? null) !== false) { + throw $refusal; + } + // Bind only the original completion after this exact refusal. + $headers[self::COMPLETION_HEADER] = $context; + return $this->uploadBlob($headers, $payload['blob'], $policy, $request['budget']); + } + } + + /** @param array $response + * @param array $expected + * @return array + */ + private function validateUploadReference(array $response, string $blob, array $expected): array + { + if (($response['schema'] ?? null) !== 'durable-workflow.v2.runtime-external-payload-upload.v1' + || ($response['transport_version'] ?? null) !== 1) { + throw new ExternalPayloadException('Unsupported runtime upload response.', 422, 'external_payload_unsupported'); + } + $reference = RuntimePayloads::validateReference($response['reference'] ?? null); + if ($reference['size_bytes'] !== strlen($blob) || !hash_equals($expected['sha256'], $reference['sha256'])) { + throw new ExternalPayloadException('Runtime upload reference differs from the submitted bytes.', 422, 'external_payload_integrity_mismatch'); + } + return $reference; + } + /** @param array $headers * @return array{threshold_bytes: int, max_bytes: int, request_bytes: int, timeout_seconds: int, status: string, completion_context?: bool, prepared_completion_context?: bool, scope_completion_context?: bool} */ @@ -412,14 +483,7 @@ private function payloads(array $body, array $slots): array $payloads = []; foreach ($slots as $segments) { $value = $this->get($body, $segments); - if (is_array($value) && array_key_exists('external_payload', $value)) { - if (count($value) !== 2 || ($value['codec'] ?? null) !== 'avro') { - throw new ExternalPayloadException('Invalid runtime payload envelope.', 422, 'external_payload_unsupported'); - } - RuntimePayloads::validateReference($value['external_payload']); - } - $blob = is_string($value) ? $value : (is_array($value) && count($value) === 2 - && ($value['codec'] ?? null) === 'avro' ? ($value['blob'] ?? null) : null); + $blob = $this->payloadBlob($value); if (is_string($blob)) { $payloads[] = ['path' => $segments, 'blob' => $blob]; } @@ -427,6 +491,22 @@ private function payloads(array $body, array $slots): array return $payloads; } + private function payloadBlob(mixed $value): ?string + { + if (is_array($value) && array_key_exists('external_payload', $value)) { + if (count($value) !== 2 || ($value['codec'] ?? null) !== 'avro') { + throw new ExternalPayloadException('Invalid runtime payload envelope.', 422, 'external_payload_unsupported'); + } + RuntimePayloads::validateReference($value['external_payload']); + return null; + } + if (is_string($value)) { + return $value; + } + return is_array($value) && count($value) === 2 && ($value['codec'] ?? null) === 'avro' + && is_string($value['blob'] ?? null) ? $value['blob'] : null; + } + /** Bound diagnostic text against complete metadata before planning ordinary uploads. * @param array $body * @param list, blob: string}> $payloads From 895d292a27227a95695dc866e87561e3d6998443 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Fri, 23 Oct 2026 16:34:21 +0000 Subject: [PATCH 19/19] Align quickstart package identities with SDK 3.5.0 --- docs/quickstart-contract.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/quickstart-contract.json b/docs/quickstart-contract.json index d98883b6..824e1d14 100644 --- a/docs/quickstart-contract.json +++ b/docs/quickstart-contract.json @@ -3,9 +3,9 @@ "schema_version": 2, "package": { "name": "durable-workflow/sdk", - "published_version": "3.4.1", - "composer_requirement": "3.4.1", - "onboarding_requirement": "^3.4" + "published_version": "3.5.0", + "composer_requirement": "3.5.0", + "onboarding_requirement": "^3.5" }, "runtime_targets": { "server": {