From 5c189681edcce3eb9545ae059bae5ee87b012f21 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 7 Nov 2026 06:33:47 +0000 Subject: [PATCH 1/3] Bind remaining completion-poll controls to reported lease authority --- phpstan.neon | 1 + .../Support/CompletionPollFaults.php | 80 +++++++++++++++++++ scripts/conformance/completion-polls.md | 24 ++++++ tests/CompletionPollConsumerFaultsTest.php | 69 ++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 scripts/conformance/Support/CompletionPollFaults.php create mode 100644 tests/CompletionPollConsumerFaultsTest.php diff --git a/phpstan.neon b/phpstan.neon index 886e888f..5ef5f541 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -7,6 +7,7 @@ parameters: - scripts/conformance/Support/CompletionPollTrace.php - scripts/conformance/Support/CompletionPollEvidence.php - scripts/conformance/Support/CompletionPollFailure.php + - scripts/conformance/Support/CompletionPollFaults.php - scripts/conformance/Support/completion-poll-relay.php excludePaths: analyse: diff --git a/scripts/conformance/Support/CompletionPollFaults.php b/scripts/conformance/Support/CompletionPollFaults.php new file mode 100644 index 00000000..37e0ce08 --- /dev/null +++ b/scripts/conformance/Support/CompletionPollFaults.php @@ -0,0 +1,80 @@ + $answer */ + public static function delay(array $answer, string $fault, float $maximumSeconds): float + { + $window = self::window($answer); + // Transit/refence cases must cross the installed SDK's ten-second + // confirmation reserve while the original lease is still live. + if ($window < 12 || !is_finite($maximumSeconds) || $maximumSeconds <= 0) { + throw new RuntimeException('completion_fault_grant_not_targetable'); + } + $delay = match ($fault) { + 'transit', 'refence' => $window - 8, + 'expiry' => $window + 0.25, + default => throw new RuntimeException('completion_fault_unknown'), + }; + if ($delay > $maximumSeconds) { + throw new RuntimeException('completion_fault_budget'); + } + return $delay; + } + + /** @param array $answer */ + public static function window(array $answer): float + { + $clock = self::timestamp($answer['server_time'] ?? null); + $expiry = self::timestamp($answer['task']['lease_expires_at'] ?? null); + if (($answer['poll_status'] ?? null) !== 'leased' || $clock === null || $expiry === null || $expiry <= $clock) { + throw new RuntimeException('completion_fault_original_grant_required'); + } + return $expiry - $clock; + } + + /** A replay may refresh its response clock, never these original task fields. + * @param array $answer + */ + public static function identity(array $answer): string + { + $task = $answer['task'] ?? null; + if (!is_array($task) || !is_string($task['task_id'] ?? null) || !is_string($task['lease_owner'] ?? null) + || !is_string($task['lease_expires_at'] ?? null)) { + throw new RuntimeException('completion_fault_original_grant_required'); + } + // Retain only the digest. History, payloads and session documents do + // not enter the public trace. Schema63's normal replay preserves the + // whole task, including additional authority fields; only its envelope + // server_time refreshes. Object key ordering is not an identity change. + return hash('sha256', json_encode(self::ordered($task), JSON_THROW_ON_ERROR)); + } + + private static function ordered(mixed $value): mixed + { + if (!is_array($value)) return $value; + if (!array_is_list($value)) ksort($value); + return array_map(self::ordered(...), $value); + } + + public static function timestamp(mixed $value): ?float + { + if (!is_string($value) + || preg_match('/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z\z/', $value) !== 1) { + return null; + } + $date = DateTimeImmutable::createFromFormat('!Y-m-d\TH:i:s.u\Z', $value, new DateTimeZone('UTC')); + return $date !== false && $date->format('Y-m-d\TH:i:s.u\Z') === $value ? (float) $date->format('U.u') : null; + } +} diff --git a/scripts/conformance/completion-polls.md b/scripts/conformance/completion-polls.md index 2853c6d9..fb4fbdd9 100644 --- a/scripts/conformance/completion-polls.md +++ b/scripts/conformance/completion-polls.md @@ -102,3 +102,27 @@ and evidence unit tests use synthetic metadata only. Relay controls forward fixed HTTP data to test byte/role forwarding, error preservation and one successful-answer loss. They create no Server, package installation, profile or published-product acceptance. + +The remaining matrix is being prepared against the same accepted schema63 +API. The initial four scenario names remain the only executable selections +until their successors are connected and qualified. Delay selection uses the +actual normal-poll `server_time` and original `lease_expires_at`. An absent +clock, unsuitable grant window or insufficient caller budget blocks that +control rather than substituting a worker clock or changing Server state. + +| Control | Actual operation and required observation | +| --- | --- | +| Carried-answer loss, each of three paths | Withhold one actual successful completion answer. Recover its original poll ID and complete task/history identity. | +| Original completion transit | Delay the real completion request before its carried claim. Require original-request charging and confirmation before callback when the reported reserve is insufficient. | +| Expired carried lease | Hold the real answer past its reported original expiry. Require fenced confirmation/refusal and no callback or publication under that expired generation. | +| Registration replacement | Re-register this invocation's same synthetic worker through the public worker API. Retain only token digests and require old-grant refusal; clean the owned replacement with its actual fenced receipt. | +| Buffered lease age | Delay a real registration-heartbeat reply after prefetch. Require workflow lease confirmation before the aged task executes. Registration heartbeat does not renew task authority. | +| Opposite slot and missing result | Hold actual opposite-kind leases through public polls/completions. Observe real carrier omission, then reservation release and later work without disabling the capability. | +| Lost ordinary prefetch | Withhold one real successful prefetch answer. Preserve the original ID, backoff and lease/history on recovery without another reservation. | +| Default OFF | Exercise omitted and explicit-false options against the actual runtime. Keep the installed fixed-socket packet regression separate from native producer acceptance. | +| Bounded load | Direct runtime calls with no relay delay or per-request trace. Compare explicit OFF/ON cells for six workers, 200 workflows and five 20ms local or remote steps. Retain all three repetitions and regressions. | + +Load cells require caller evidence for fresh isolated databases and the actual +backend, image, host and resource settings. The runner does not create a +database, invent an administrative reset or infer backend identity from a +label. Descriptive timings alone are not a speedup or release acceptance. diff --git a/tests/CompletionPollConsumerFaultsTest.php b/tests/CompletionPollConsumerFaultsTest.php new file mode 100644 index 00000000..2d522b72 --- /dev/null +++ b/tests/CompletionPollConsumerFaultsTest.php @@ -0,0 +1,69 @@ +expectExceptionMessage('completion_fault_budget'); + CompletionPollFaults::delay(self::grant(), 'expiry', 30); + } + + public function testMissingProducerClockDoesNotBecomeAWorkerWallClock(): void + { + $grant = self::grant(); + unset($grant['server_time']); + $this->expectExceptionMessage('completion_fault_original_grant_required'); + CompletionPollFaults::delay($grant, 'transit', 60); + } + + public function testAnAlreadyExpiredGrantCannotQualifyTheLiveTransitCase(): void + { + $grant = self::grant(); + $grant['task']['lease_expires_at'] = $grant['server_time']; + $this->expectExceptionMessage('completion_fault_original_grant_required'); + CompletionPollFaults::delay($grant, 'transit', 60); + } + + public function testFreshReplayClockDoesNotAlterOriginalLeaseIdentity(): void + { + $first = self::grant(); + $replay = $first; + $replay['server_time'] = '2026-11-07T08:00:05.000000Z'; + self::assertSame(CompletionPollFaults::identity($first), CompletionPollFaults::identity($replay)); + $replay['task']['lease_expires_at'] = '2026-11-07T08:00:35.000000Z'; + self::assertNotSame(CompletionPollFaults::identity($first), CompletionPollFaults::identity($replay)); + } + + public function testReplayHistoryAndAttemptChangesRemainVisibleWithoutRetainingPayloads(): void + { + $first = self::grant(); + foreach (['workflow_task_attempt' => 2, 'history_events' => [['changed' => true]], 'ready_age_ms' => 1] as $field => $value) { + $changed = $first; + $changed['task'][$field] = $value; + self::assertNotSame(CompletionPollFaults::identity($first), CompletionPollFaults::identity($changed)); + } + } + + private static function grant(): array + { + return ['poll_status' => 'leased', 'server_time' => '2026-11-07T08:00:00.000000Z', + 'task' => ['task_id' => 'fixture-task', 'lease_owner' => 'fixture-worker', 'workflow_task_attempt' => 1, + 'lease_expires_at' => '2026-11-07T08:00:30.000000Z', 'ready_age_ms' => 0, 'history_events' => []]]; + } +} From 55dd1b82bf32c8648f8bd28495c43295faadefeb Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 7 Nov 2026 07:08:51 +0000 Subject: [PATCH 2/3] Connect completion-poll failure and capacity qualification controls --- phpstan.neon | 4 + .../Support/CompletionPollEvidence.php | 4 + .../Support/CompletionPollFaults.php | 4 +- .../Support/CompletionPollMatrixEvidence.php | 264 ++++++++++++++ .../Support/CompletionPollOppositeSlots.php | 164 +++++++++ .../Support/CompletionPollPacketDefaults.php | 63 ++++ .../Support/CompletionPollRelayFaults.php | 146 ++++++++ .../Support/CompletionPollSubject.php | 4 +- .../Support/CompletionPollTrace.php | 21 +- .../Support/completion-poll-relay.php | 10 +- scripts/conformance/completion-polls.md | 40 ++- .../conformance/qualify-completion-polls.php | 166 +++++++-- tests/CompletionPollConsumerFaultsTest.php | 1 - ...mpletionPollConsumerMatrixEvidenceTest.php | 329 ++++++++++++++++++ ...ompletionPollConsumerOppositeSlotsTest.php | 34 ++ ...mpletionPollConsumerPacketDefaultsTest.php | 39 +++ .../CompletionPollConsumerRelayFaultsTest.php | 66 ++++ 17 files changed, 1304 insertions(+), 55 deletions(-) create mode 100644 scripts/conformance/Support/CompletionPollMatrixEvidence.php create mode 100644 scripts/conformance/Support/CompletionPollOppositeSlots.php create mode 100644 scripts/conformance/Support/CompletionPollPacketDefaults.php create mode 100644 scripts/conformance/Support/CompletionPollRelayFaults.php create mode 100644 tests/CompletionPollConsumerMatrixEvidenceTest.php create mode 100644 tests/CompletionPollConsumerOppositeSlotsTest.php create mode 100644 tests/CompletionPollConsumerPacketDefaultsTest.php create mode 100644 tests/CompletionPollConsumerRelayFaultsTest.php diff --git a/phpstan.neon b/phpstan.neon index 5ef5f541..0461ed17 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -8,6 +8,10 @@ parameters: - scripts/conformance/Support/CompletionPollEvidence.php - scripts/conformance/Support/CompletionPollFailure.php - scripts/conformance/Support/CompletionPollFaults.php + - scripts/conformance/Support/CompletionPollRelayFaults.php + - scripts/conformance/Support/CompletionPollMatrixEvidence.php + - scripts/conformance/Support/CompletionPollPacketDefaults.php + - scripts/conformance/Support/CompletionPollOppositeSlots.php - scripts/conformance/Support/completion-poll-relay.php excludePaths: analyse: diff --git a/scripts/conformance/Support/CompletionPollEvidence.php b/scripts/conformance/Support/CompletionPollEvidence.php index d3c33e1c..c33ca754 100644 --- a/scripts/conformance/Support/CompletionPollEvidence.php +++ b/scripts/conformance/Support/CompletionPollEvidence.php @@ -7,6 +7,7 @@ use RuntimeException; require_once __DIR__.'/CompletionPollFailure.php'; +require_once __DIR__.'/CompletionPollMatrixEvidence.php'; /** Assertions over actual relay observations, not a producer or source-test acceptance substitute. */ final class CompletionPollEvidence @@ -19,6 +20,9 @@ final class CompletionPollEvidence */ public static function verify(array $events, string $scenario, string $worker, string $prefix): array { + if (in_array($scenario, CompletionPollMatrixEvidence::SCENARIOS, true)) { + return CompletionPollMatrixEvidence::verify($events, $scenario, $worker, $prefix); + } if ($events === [] || !in_array($scenario, CompletionPollSubject::SCENARIOS, true)) { throw new RuntimeException('completion_evidence_empty'); } diff --git a/scripts/conformance/Support/CompletionPollFaults.php b/scripts/conformance/Support/CompletionPollFaults.php index 37e0ce08..faf3dfa8 100644 --- a/scripts/conformance/Support/CompletionPollFaults.php +++ b/scripts/conformance/Support/CompletionPollFaults.php @@ -17,13 +17,13 @@ final class CompletionPollFaults public static function delay(array $answer, string $fault, float $maximumSeconds): float { $window = self::window($answer); - // Transit/refence cases must cross the installed SDK's ten-second + // Transit cases must cross the installed SDK's ten-second // confirmation reserve while the original lease is still live. if ($window < 12 || !is_finite($maximumSeconds) || $maximumSeconds <= 0) { throw new RuntimeException('completion_fault_grant_not_targetable'); } $delay = match ($fault) { - 'transit', 'refence' => $window - 8, + 'transit' => $window - 8, 'expiry' => $window + 0.25, default => throw new RuntimeException('completion_fault_unknown'), }; diff --git a/scripts/conformance/Support/CompletionPollMatrixEvidence.php b/scripts/conformance/Support/CompletionPollMatrixEvidence.php new file mode 100644 index 00000000..d8665d0c --- /dev/null +++ b/scripts/conformance/Support/CompletionPollMatrixEvidence.php @@ -0,0 +1,264 @@ +> $events + * @return array + */ + public static function verify(array $events, string $scenario, string $worker, string $prefix): array + { + $requests = $answers = $deliveries = $current = $callbacks = []; + foreach ($events as $position => $event) { + if (in_array($event['kind'], ['relay_error', 'cleanup_error'], true)) throw new RuntimeException('completion_evidence_process_failure'); + if ($event['kind'] === 'worker_error' && !in_array($scenario, ['carrier-expiry', 'carrier-refence'], true)) { + throw new RuntimeException('completion_evidence_process_failure'); + } + $event['position'] = $position; + if ($event['kind'] === 'request') $requests[$event['request']] = $event; + if ($event['kind'] === 'upstream') $answers[$event['request']] = $event; + if ($event['kind'] === 'delivered') { + $deliveries[$event['request']] = $event; + if ($event['status'] !== 200) continue; + $answer = $answers[$event['request']]['answer'] ?? []; + foreach ([$answer['task'] ?? null, $answer['activity_poll']['task'] ?? null, $answer['workflow_poll']['task'] ?? null] as $task) { + if (!is_array($task)) continue; + $current[$task['workflow_id']] = $task; + $current[$task['task_id']] = $task; + } + } + if (in_array($event['kind'], ['callback', 'activity'], true)) { + $task = $current[$event[$event['kind'] === 'callback' ? 'workflow_id' : 'task_id']] ?? null; + if (is_array($task)) $callbacks[self::key($task)][] = $event; + } + } + if ($requests === [] || $answers === []) throw new RuntimeException('completion_evidence_empty'); + if ($scenario === 'capacity') { + CompletionPollEvidence::verify($events, 'carriers', $worker, $prefix); + foreach ($answers as $id => $answer) { + $body = $requests[$id]['body']; + if (!isset($body['activity_poll'], $body['workflow_poll']) + || $answer['status'] !== 200 || ($deliveries[$id]['status'] ?? null) !== 200 + || ($answer['answer']['activity_poll']['leased'] ?? null) !== true + || array_key_exists('workflow_poll', $answer['answer'])) continue; + foreach ($requests as $nextId => $next) { + if ($next['position'] > $deliveries[$id]['position'] + && isset($next['body']['workflow_poll']) + && $next['body']['workflow_poll']['poll_request_id'] !== $body['workflow_poll']['poll_request_id'] + && ($answers[$nextId]['status'] ?? null) === 200 && ($deliveries[$nextId]['status'] ?? null) === 200 + && ($answers[$nextId]['answer']['workflow_poll']['leased'] ?? null) === true) { + return ['missing_key_released_reservation' => true, 'capability_remained_enabled' => true]; + } + } + } + throw CompletionPollFailure::product('completion_evidence_missing_key_capacity'); + } + if (in_array($scenario, ['default-off', 'explicit-off'], true)) { + $option = self::fault($events, 'default_option', null); + self::require($option['default'] === false && $option['explicit'] === ($scenario === 'explicit-off'), + 'completion_evidence_default_option'); + $parents = $children = []; + foreach ($answers as $event) { + foreach ([$event['answer']['task'] ?? null, $event['answer']['workflow_poll']['task'] ?? null] as $task) { + if (($task['workflow_id'] ?? null) === $prefix.'-2') $parents[] = $task['task_id']; + if (($task['workflow_id'] ?? null) === $prefix.'-3') $children[] = $task; + } + } + foreach ($requests as $id => $request) { + foreach (array_unique($parents) as $parent) { + if (!str_ends_with($request['path'], '/workflow-tasks/'.$parent.'/complete')) continue; + foreach ($children as $child) { + foreach ($callbacks[self::key($child)] ?? [] as $callback) { + self::require(($deliveries[$id]['status'] ?? null) === 200 + && $deliveries[$id]['position'] < $callback['position'], 'completion_evidence_default_off_order'); + return ['completion_remained_synchronous' => true, 'option' => $scenario]; + } + } + } + } + throw new RuntimeException('completion_evidence_default_off_target_missing'); + } + if ($scenario === 'pipeline-buffer-age') { + CompletionPollEvidence::verify($events, 'pipeline', $worker, $prefix); + $hold = self::fault($events, 'fault_begin', 'buffer_age'); + $end = self::fault($events, 'fault_end', 'buffer_age'); + self::require(($end['at'] - $hold['at']) / 1e9 >= 1, 'completion_evidence_buffer_age'); + foreach ($answers as $id => $answer) { + $task = $answer['answer']['task'] ?? null; + if (($task['workflow_id'] ?? null) !== $prefix.'-3' || !isset($deliveries[$id])) continue; + self::require($deliveries[$id]['position'] < $hold['position'], 'completion_evidence_buffer_age'); + self::confirmation($task, $callbacks, $requests, $answers, $deliveries, $end['position']); + return ['actual_buffer_age_seconds' => ($end['at'] - $hold['at']) / 1e9, 'confirmed_before_callback' => true]; + } + throw new RuntimeException('completion_evidence_buffer_target_missing'); + } + if ($scenario === 'completion-transit') { + $fault = self::fault($events, 'fault_begin', 'completion_transit'); + $id = $fault['request']; + $grant = $answers[$id]['answer']['workflow_poll'] ?? []; + $window = CompletionPollFaults::window($grant); + $transit = ($deliveries[$id]['at'] - $requests[$id]['at']) / 1e9; + self::require(($answers[$id]['status'] ?? null) === 200 && $window >= 12 && $window - $transit < 10, + 'completion_evidence_completion_transit_not_targeted'); + self::confirmation($grant['task'], $callbacks, $requests, $answers, $deliveries, $deliveries[$id]['position']); + return ['original_completion_transit_seconds' => $transit, 'confirmed_before_callback' => true]; + } + if ($scenario === 'carrier-expiry') { + $fault = self::fault($events, 'fault_begin', 'expiry'); + $id = $fault['request']; + $grant = $answers[$id]['answer']['workflow_poll'] ?? []; + $task = $grant['task'] ?? throw new RuntimeException('completion_evidence_expiry_target_missing'); + self::require(($deliveries[$id]['at'] - $answers[$id]['at']) / 1e9 >= CompletionPollFaults::window($grant), + 'completion_evidence_expiry_not_reached'); + self::require(($callbacks[self::key($task)] ?? []) === [], 'completion_evidence_expired_callback'); + $refused = false; + foreach ($requests as $requestId => $request) { + if (!self::fence($request, $task)) continue; + $reply = $answers[$requestId] ?? []; + if (str_ends_with($request['path'], '/heartbeat') && ($reply['status'] ?? null) === 409 + && in_array($reply['answer']['reason'] ?? null, ['lease_expired', 'task_not_leased', 'workflow_task_attempt_mismatch'], true) + && ($reply['answer']['task_id'] ?? null) === $task['task_id'] + && ($reply['answer']['workflow_task_attempt'] ?? null) === $task['workflow_task_attempt']) $refused = true; + if (str_ends_with($request['path'], '/complete') || str_ends_with($request['path'], '/fail')) { + self::require(($reply['answer']['recorded'] ?? null) !== true, 'completion_evidence_expired_publication'); + } + } + self::require($refused, 'completion_evidence_expiry_refusal_missing'); + return ['original_expiry_reached' => true, 'expired_callback_or_publication' => false, 'real_identity_bound_refusal' => true]; + } + $refence = $scenario === 'carrier-refence'; + $prefetch = $scenario === 'prefetch-loss'; + $fault = self::fault($events, 'fault', $refence ? 'refenced_activity_answer' : ($prefetch ? 'lost_prefetch_answer' : 'lost_carried_answer')); + $id = $fault['request']; + $parent = $requests[$id]; + $member = $fault['member'] ?? null; + $kind = $member === 'activity_poll' ? 'activity' : 'workflow'; + $original = $prefetch ? $answers[$id]['answer'] : ($answers[$id]['answer'][$member] ?? []); + $task = $original['task'] ?? throw new RuntimeException('completion_evidence_loss_target_missing'); + $poll = $prefetch ? $parent['body'] : ($parent['body'][$member] ?? []); + $expectedMember = $scenario === 'carrier-loss-activity' || $refence ? 'activity_poll' : 'workflow_poll'; + $expectedPath = $scenario === 'carrier-loss-workflow' ? '/activity-tasks/' : '/workflow-tasks/'; + self::require(($answers[$id]['status'] ?? null) === 200 && ($deliveries[$id]['status'] ?? null) === 502 + && ($prefetch || ($member === $expectedMember && str_contains($parent['path'], $expectedPath))) + && ($poll['worker_id'] ?? null) === $worker && ($task['lease_owner'] ?? null) === $worker + && is_string($poll['poll_request_id'] ?? null), 'completion_evidence_loss_not_targeted'); + foreach ($requests as $retryId => $request) { + if ($request['position'] <= $deliveries[$id]['position'] + || !str_ends_with($request['path'], '/'.$kind.'-tasks/poll')) continue; + self::require(($request['body']['poll_request_id'] ?? null) === $poll['poll_request_id'], 'completion_evidence_recovery_identity_changed'); + if ($refence) { + $rotation = self::fault($events, 'refence', null); + self::require($rotation['old_registration_sha256'] !== $rotation['registration_sha256'] + && $rotation['position'] < $request['position'] && ($answers[$retryId]['status'] ?? null) === 409 + && ($deliveries[$retryId]['status'] ?? null) === 409 + && ($answers[$retryId]['answer']['reason'] ?? null) === 'poll_cached_task_expired' + && ($callbacks[self::key($task)] ?? []) === [], 'completion_evidence_refence_recovery'); + return ['registration_replaced' => true, 'original_activity_recovery_refused' => true, + 'existing_workflow_grant_revocation_asserted' => false]; + } + $replayed = $answers[$retryId]['answer']['task'] ?? null; + self::require(($answers[$retryId]['status'] ?? null) === 200 && is_array($replayed) + && is_string($task['identity_sha256'] ?? null) + && $task['identity_sha256'] === ($replayed['identity_sha256'] ?? null), 'completion_evidence_replay_authority_changed'); + self::require(($deliveries[$retryId]['status'] ?? null) === 200 + && array_filter($callbacks[self::key($task)] ?? [], static fn (array $event): bool => $event['position'] > $deliveries[$retryId]['position']) !== [], + 'completion_evidence_recovered_callback_missing'); + if (!$prefetch) { + $sameParent = array_filter($requests, static fn (array $candidate): bool => $candidate['path'] === $parent['path']); + self::require(count($sameParent) >= 2, 'completion_evidence_completion_retry_missing'); + foreach ($sameParent as $candidate) { + self::require($candidate['body'] === $parent['body'], 'completion_evidence_completion_retry_changed'); + } + } else { + $parentTasks = []; + foreach ($answers as $answer) { + foreach ([$answer['answer']['task'] ?? null, $answer['answer']['workflow_poll']['task'] ?? null] as $candidate) { + if (($candidate['workflow_id'] ?? null) === $prefix.'-2') $parentTasks[] = $candidate['task_id']; + } + } + $published = false; + foreach ($requests as $parentId => $candidate) { + foreach (array_unique($parentTasks) as $taskId) { + if (str_ends_with($candidate['path'], '/workflow-tasks/'.$taskId.'/complete') + && $candidate['position'] > $parent['position'] && $candidate['position'] < $request['position'] + && ($answers[$parentId]['status'] ?? null) === 200 && !isset($candidate['body']['workflow_poll'])) $published = true; + } + } + $retry = array_filter($events, static fn (array $event): bool => $event['kind'] === 'poll_retry' + && $event['task_kind'] === 'workflow' && $event['attempt'] === 1 && $event['delay'] >= 0.1); + self::require($published && $retry !== [] + && ($request['at'] - $deliveries[$id]['at']) / 1e9 >= 0.09, 'completion_evidence_prefetch_recovery_policy'); + } + return ['original_poll_id_recovered' => true, 'original_task_history_deadlines_preserved' => true, 'kind' => $kind]; + } + throw CompletionPollFailure::product('completion_evidence_recovery_missing'); + } + + /** @param list> $events + * @return array + */ + private static function fault(array $events, string $kind, ?string $mode): array + { + foreach ($events as $position => $event) { + if ($event['kind'] === $kind && ($mode === null || ($event['mode'] ?? null) === $mode)) return $event + ['position' => $position]; + } + throw new RuntimeException('completion_evidence_fault_not_observed'); + } + + /** @param array $task */ + private static function key(array $task): string + { + return json_encode([$task['task_id'] ?? null, $task['lease_owner'] ?? null, + $task['workflow_task_attempt'] ?? null, $task['activity_attempt_id'] ?? null], JSON_THROW_ON_ERROR); + } + + /** @param array $request + * @param array $task + */ + private static function fence(array $request, array $task): bool + { + return str_contains($request['path'], '/workflow-tasks/'.rawurlencode($task['task_id']).'/') + && ($request['body']['lease_owner'] ?? null) === $task['lease_owner'] + && ($request['body']['workflow_task_attempt'] ?? null) === $task['workflow_task_attempt']; + } + + /** @param array $task + * @param array>> $callbacks + * @param array> $requests + * @param array> $answers + * @param array> $deliveries + */ + private static function confirmation(array $task, array $callbacks, array $requests, array $answers, array $deliveries, int $after): void + { + foreach ($callbacks[self::key($task)] ?? [] as $callback) { + foreach ($requests as $id => $request) { + if (self::fence($request, $task) && str_ends_with($request['path'], '/heartbeat') + && $request['position'] > $after && ($answers[$id]['status'] ?? null) === 200 + && ($answers[$id]['answer']['renewed'] ?? null) === true + && ($answers[$id]['answer']['task_id'] ?? null) === $task['task_id'] + && ($answers[$id]['answer']['lease_owner'] ?? null) === $task['lease_owner'] + && ($answers[$id]['answer']['workflow_task_attempt'] ?? null) === $task['workflow_task_attempt'] + && ($deliveries[$id]['status'] ?? null) === 200 && $deliveries[$id]['position'] < $callback['position']) return; + } + } + throw CompletionPollFailure::product('completion_evidence_confirmation_missing'); + } + + private static function require(bool $condition, string $code): void + { + if (!$condition) throw CompletionPollFailure::product($code); + } +} diff --git a/scripts/conformance/Support/CompletionPollOppositeSlots.php b/scripts/conformance/Support/CompletionPollOppositeSlots.php new file mode 100644 index 00000000..31b2cf50 --- /dev/null +++ b/scripts/conformance/Support/CompletionPollOppositeSlots.php @@ -0,0 +1,164 @@ + */ + public static function run(CompletionPollSubject $subject, string $clientToken, string $workerToken): array + { + return CompletionPipelineScenario::run(static function (string $relay, string $file) use ($subject, $clientToken, $workerToken): array { + $data = $subject->data; + $prefix = 'opposite-'.bin2hex(random_bytes(8)); + $worker = $prefix.'-worker'; + $queue = $prefix.'-queue'; + file_put_contents($file, json_encode(['events' => [], 'endpoint' => $data['endpoint'], + 'scenario' => 'opposite-slots', 'prefix' => $prefix], JSON_THROW_ON_ERROR)); + $trace = new CompletionPollTrace($file); + $control = new Client($relay, namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 2); + $actor = new Client($relay, namespace: $data['namespace'], workerToken: $workerToken, workerRequestTimeoutSeconds: 2); + $cleanup = new Client($data['endpoint'], namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 1); + $deadline = hrtime(true) / 1e9 + $data['deadline_seconds']; + $check = static function () use ($deadline): void { + if (hrtime(true) / 1e9 >= $deadline) throw new RuntimeException('completion_opposite_deadline'); + }; + $token = null; + $failure = null; + $assertions = []; + $terminated = 0; + $handles = []; + for ($i = 1; $i <= 3; ++$i) $handles[$i] = $control->workflowHandle($prefix.'-'.$i); + try { + if ($control->clusterInfo()->version !== $data['server_version']) throw new RuntimeException('completion_runtime_version'); + $registered = $actor->registerWorker($worker, $queue, ['qualification.wire'], ['qualification.remote'], + maxConcurrentWorkflowTasks: 2, maxConcurrentActivityTasks: 2); + $token = $registered['registration_token'] ?? null; + if (($registered['registered'] ?? null) !== true || ($registered['worker_id'] ?? null) !== $worker + || ($registered['task_queue'] ?? null) !== $queue) throw new RuntimeException('completion_opposite_registration_required'); + foreach (CompletionPollEvidence::CAPABILITIES as $capability) { + if (($registered['server_capabilities'][$capability] ?? null) !== true) throw new RuntimeException('completion_opposite_capability_required'); + } + if (!is_string($token) || preg_match('/\A[a-f0-9]{32}\z/', $token) !== 1) throw new RuntimeException('completion_refence_receipt_required'); + foreach ($handles as $index => $handle) { + $check(); + $handles[$index] = $control->startWorkflow('qualification.wire', $handle->workflowId, $queue, + input: [$index], executionTimeoutSeconds: 180, runTimeoutSeconds: 150); + } + $workflowOne = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf1'), $worker, 'workflow'); + $workflowTwo = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf2'), $worker, 'workflow'); + if ($workflowOne['task_id'] === $workflowTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); + $check(); + $activityId = $prefix.'-carried-activity'; + $answer = $actor->completeWorkflowTask($workflowOne['task_id'], $worker, $workflowOne['workflow_task_attempt'], + self::schedule($actor), activityPoll: self::carrier($worker, $queue, $activityId)); + self::omitted($answer, 'activity_poll'); + // The other original workflow is still leased. A same-ID + // ordinary activity poll proves eligible work was present. + $activityOne = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $activityId), $worker, 'activity'); + $actor->completeWorkflowTask($workflowTwo['task_id'], $worker, $workflowTwo['workflow_task_attempt'], self::schedule($actor)); + $activityTwo = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $prefix.'-act2'), $worker, 'activity'); + if ($activityOne['task_id'] === $activityTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); + $check(); + $workflowId = $prefix.'-carried-workflow'; + $workflowCarrier = self::carrier($worker, $queue, $workflowId) + ['history_page_size' => 100, 'task_kinds' => ['workflow']]; + $answer = $actor->completeActivityTask($activityTwo['task_id'], $activityTwo['activity_attempt_id'], $worker, 2, + workflowPoll: $workflowCarrier); + self::omitted($answer, 'workflow_poll'); + // There is both a third original workflow and the activity's + // next workflow task ready. Release the held activity normally, + // then recover with the omitted carrier's exact original ID. + $actor->completeActivityTask($activityOne['task_id'], $activityOne['activity_attempt_id'], $worker, 1); + $recovered = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $workflowId), $worker, 'workflow'); + $assertions = ['scope' => 'installed Client public-API opposite-slot control; managed Worker capacity is a separate row', + 'live_workflow_declined_activity_carrier' => true, 'live_activity_declined_workflow_carrier' => true, + 'eligible_work_and_original_id_followup' => true, 'recovered_workflow_task' => CompletionPollTrace::task($recovered)]; + } catch (Throwable $error) { + $failure = $error; + } finally { + $cleanupDeadline = hrtime(true) / 1e9 + 25; + foreach ($handles as $handle) { + try { + if (hrtime(true) / 1e9 >= $cleanupDeadline) throw new RuntimeException('completion_opposite_cleanup_deadline'); + $description = $cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId); + if (!in_array($description->status, ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { + $cleanup->terminateWorkflow($handle->workflowId, 'Opposite-slot qualification cleanup', $handle->selectedRunId); + ++$terminated; + $closed = $cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId); + if (!in_array($closed->status, ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { + throw new RuntimeException('completion_opposite_cleanup_unconfirmed'); + } + } + } catch (Throwable $error) { + if ($error instanceof ServerException && $error->status === 404 && $handle->selectedRunId === null) continue; + $trace->record(['kind' => 'cleanup_error']); + $failure = new RuntimeException('completion_opposite_cleanup_failed', previous: $error); + } + } + if (is_string($token)) { + try { + $actor->deregisterWorkerRegistration($worker, $token); + } catch (Throwable $error) { + $failure = new RuntimeException('completion_opposite_cleanup_failed', previous: $error); + } + } + } + if ($failure === null && ($cleanup->listWorkers($queue)['workers'] ?? []) !== []) { + $failure = new RuntimeException('completion_worker_registration_survived'); + } + return ['scenario' => 'opposite-slots', 'outcome' => $failure === null ? 'pass' : CompletionPollFailure::diagnostic($failure)['outcome'], + 'diagnostic' => $failure === null ? null : CompletionPollFailure::diagnostic($failure), + 'assertions' => $assertions, 'workflows_intentionally_terminated' => $terminated, 'events' => $trace->state()['events']]; + }, __DIR__.'/completion-poll-relay.php', true); + } + + /** @return array */ + private static function carrier(string $worker, string $queue, string $id): array + { + return ['worker_id' => $worker, 'task_queue' => $queue, 'poll_request_id' => $id, 'timeout_seconds' => 0]; + } + + /** @return list> */ + private static function schedule(Client $client): array + { + return [['type' => 'schedule_activity', 'activity_type' => 'qualification.remote', + 'arguments' => $client->payloadCodec()->envelope([1]), 'start_to_close_timeout' => 120]]; + } + + /** @param array $answer + * @return array + */ + private static function claim(array $answer, string $worker, string $kind): array + { + $task = $answer['task'] ?? null; + if (($answer['poll_status'] ?? null) !== 'leased' || !is_array($task) + || ($task['lease_owner'] ?? null) !== $worker || ($task['task_kind'] ?? null) !== $kind + || !is_string($task['task_id'] ?? null) || !is_string($task['lease_expires_at'] ?? null) + || ($kind === 'workflow' && (!is_int($task['workflow_task_attempt'] ?? null) || $task['workflow_task_attempt'] < 1)) + || ($kind === 'activity' && (!is_string($task['activity_attempt_id'] ?? null) || $task['activity_attempt_id'] === ''))) { + throw CompletionPollFailure::product('completion_opposite_claim_required'); + } + return $task; + } + + /** @param array $answer */ + public static function omitted(array $answer, string $member): void + { + if (($answer['recorded'] ?? null) !== true || array_key_exists($member, $answer)) { + throw CompletionPollFailure::product('completion_opposite_carrier_not_declined'); + } + } +} diff --git a/scripts/conformance/Support/CompletionPollPacketDefaults.php b/scripts/conformance/Support/CompletionPollPacketDefaults.php new file mode 100644 index 00000000..ffe7f93f --- /dev/null +++ b/scripts/conformance/Support/CompletionPollPacketDefaults.php @@ -0,0 +1,63 @@ + */ + public static function run(): array + { + foreach (['__construct', 'create'] as $method) { + $options = array_filter((new ReflectionMethod(Worker::class, $method))->getParameters(), + static fn ($parameter): bool => $parameter->getName() === 'enableCompletionPipelining'); + if (count($options) !== 1 || current($options)->getDefaultValue() !== false) { + throw CompletionPollFailure::product('completion_default_option_not_false'); + } + } + $implicit = WorkerPipelineScenario::run(null); + $explicit = WorkerPipelineScenario::run(false); + if ($implicit['error'] !== null || $explicit['error'] !== null) throw new RuntimeException('completion_packet_fixture_failed'); + $first = self::digest($implicit['requests']); + $second = self::digest($explicit['requests']); + if ($first !== $second) throw CompletionPollFailure::product('completion_default_packets_changed'); + return ['outcome' => 'pass', 'runtime_observed' => false, + 'scope' => 'installed SDK fixed-socket packet regression; no Native producer acceptance', + 'normalized_packets_sha256' => $first, 'implicit_request_count' => count($implicit['requests']), + 'explicit_request_count' => count($explicit['requests']), + 'normalization' => ['generated poll identities preserve equality and reuse', 'local attempt duration_ms'], + 'default_option_false' => true]; + } + + /** @param list> $requests */ + public static function digest(array $requests): string + { + $identities = []; + $normalize = static function (mixed $value) use (&$normalize, &$identities): mixed { + if (!is_array($value)) return $value; + foreach ($value as $key => $item) { + if ($key === 'poll_request_id' && is_string($item)) { + $identities[$item] ??= 'poll-'.(count($identities) + 1); + $value[$key] = $identities[$item]; + } elseif ($key === 'duration_ms') { + $value[$key] = ''; + } else { + $value[$key] = $normalize($item); + } + } + return $value; + }; + return hash('sha256', json_encode($normalize($requests), JSON_THROW_ON_ERROR)); + } +} diff --git a/scripts/conformance/Support/CompletionPollRelayFaults.php b/scripts/conformance/Support/CompletionPollRelayFaults.php new file mode 100644 index 00000000..4f6332a3 --- /dev/null +++ b/scripts/conformance/Support/CompletionPollRelayFaults.php @@ -0,0 +1,146 @@ + $body */ + public static function before(CompletionPollTrace $trace, string $path, array $body, string $request): void + { + $state = $trace->state(); + if ($state['scenario'] !== 'completion-transit' || isset($state['fault_request']) + || !str_contains($path, '/workflow-tasks/') || !str_ends_with($path, '/complete') + || !isset($body['workflow_poll']) + || !in_array('complete_workflow', array_column($body['commands'] ?? [], 'type'), true)) { + return; + } + $grant = self::parentGrant($state, $path); + if ($grant === null) throw new RuntimeException('completion_fault_original_grant_required'); + $delay = CompletionPollFaults::delay($grant, 'transit', self::budget($state)); + if (self::select($trace, $request)) self::hold($trace, $request, 'completion_transit', $delay); + } + + /** @param array $answer + * @return bool True only when this invocation deliberately withholds a successful answer. + */ + public static function after(CompletionPollTrace $trace, string $path, array $answer, int $status, string $request): bool + { + if ($status !== 200) return false; + $state = $trace->state(); + if (isset($state['fault_request'])) return false; + $scenario = $state['scenario']; + $member = str_ends_with($path, '/complete') ? match ($scenario) { + 'carrier-loss-activity' => str_contains($path, '/workflow-tasks/') ? 'activity_poll' : null, + 'carrier-loss-workflow' => str_contains($path, '/activity-tasks/') ? 'workflow_poll' : null, + 'carrier-loss-workflow-completion' => str_contains($path, '/workflow-tasks/') ? 'workflow_poll' : null, + 'carrier-expiry' => 'workflow_poll', + 'carrier-refence' => str_contains($path, '/workflow-tasks/') ? 'activity_poll' : null, + default => null, + } : null; + $grant = $member !== null && ($answer[$member]['leased'] ?? null) === true + && is_array($answer[$member]['answer']['task'] ?? null) ? $answer[$member]['answer'] : null; + if ($grant !== null && in_array($scenario, ['carrier-loss-activity', 'carrier-loss-workflow', 'carrier-loss-workflow-completion'], true)) { + if (!self::select($trace, $request)) return false; + $trace->record(['kind' => 'fault', 'request' => $request, 'mode' => 'lost_carried_answer', 'member' => $member]); + return true; + } + if ($scenario === 'prefetch-loss' && str_ends_with($path, '/workflow-tasks/poll') + && ($answer['task']['workflow_id'] ?? null) === $state['prefix'].'-3' && self::select($trace, $request)) { + $trace->record(['kind' => 'fault', 'request' => $request, 'mode' => 'lost_prefetch_answer']); + return true; + } + if ($grant !== null && $scenario === 'carrier-refence') { + if (!self::select($trace, $request)) return false; + $trace->change(static function (array &$data): void { $data['refence_ready'] = true; }); + while (!(($trace->state())['refence_complete'] ?? false)) { + self::budget($trace->state()); + usleep(5000); + } + // The accepted vector fences original activity receipt recovery. + // It does not assert revocation of an already granted workflow. + $trace->record(['kind' => 'fault', 'request' => $request, 'mode' => 'refenced_activity_answer', 'member' => 'activity_poll']); + return true; + } + if ($grant !== null && $scenario === 'carrier-expiry') { + $delay = CompletionPollFaults::delay($grant, 'expiry', self::budget($state)); + if (!self::select($trace, $request)) return false; + self::hold($trace, $request, 'expiry', $delay); + return false; + } + if ($scenario === 'pipeline-buffer-age' && str_ends_with($path, '/worker/heartbeat') + && self::prefetchDelivered($state) && self::select($trace, $request)) { + if (self::budget($state) < CompletionPollFaults::BUFFER_AGE_SECONDS) throw new RuntimeException('completion_fault_budget'); + self::hold($trace, $request, 'buffer_age', CompletionPollFaults::BUFFER_AGE_SECONDS); + } + return false; + } + + /** @param array $state */ + private static function budget(array $state): float + { + $remaining = ($state['deadline_at'] ?? 0) - hrtime(true) / 1e9 - 5; + if ($remaining <= 0) throw new RuntimeException('completion_fault_budget'); + return min(90, $remaining); + } + + private static function select(CompletionPollTrace $trace, string $request): bool + { + $selected = false; + $trace->change(static function (array &$state) use ($request, &$selected): void { + if (!isset($state['fault_request'])) { + $state['fault_request'] = $request; + $selected = true; + } + }); + return $selected; + } + + private static function hold(CompletionPollTrace $trace, string $request, string $mode, float $seconds): void + { + $trace->record(['kind' => 'fault_begin', 'request' => $request, 'mode' => $mode, 'seconds' => $seconds]); + $until = hrtime(true) / 1e9 + $seconds; + while (hrtime(true) / 1e9 < $until) usleep(5000); + $trace->record(['kind' => 'fault_end', 'request' => $request, 'mode' => $mode]); + } + + /** @param array $state */ + private static function prefetchDelivered(array $state): bool + { + $request = null; + $delivered = false; + foreach ($state['events'] as $event) { + if ($event['kind'] === 'callback' && ($event['index'] ?? null) === 3) return false; + if ($event['kind'] === 'upstream' && ($event['answer']['task']['workflow_id'] ?? null) === $state['prefix'].'-3') { + $request = $event['request']; + } + if ($event['kind'] === 'delivered' && $event['request'] === $request && $event['status'] === 200) $delivered = true; + } + return $delivered; + } + + /** @param array $state + * @return array|null + */ + private static function parentGrant(array $state, string $path): ?array + { + $found = null; + foreach ($state['events'] as $event) { + if ($event['kind'] !== 'upstream' || ($event['status'] ?? null) !== 200) continue; + foreach ([$event['answer'], $event['answer']['workflow_poll'] ?? []] as $grant) { + if (is_string($grant['task']['task_id'] ?? null) + && str_ends_with($path, '/workflow-tasks/'.rawurlencode($grant['task']['task_id']).'/complete')) { + $found = $grant; + } + } + } + return $found; + } +} diff --git a/scripts/conformance/Support/CompletionPollSubject.php b/scripts/conformance/Support/CompletionPollSubject.php index 32f38d2a..c46bf51f 100644 --- a/scripts/conformance/Support/CompletionPollSubject.php +++ b/scripts/conformance/Support/CompletionPollSubject.php @@ -12,7 +12,9 @@ final class CompletionPollSubject { public const SCHEMA = 'durable-workflow.php-completion-polls-subject/v1'; - public const SCENARIOS = ['carriers', 'pipeline', 'pipeline-loss', 'fallback']; + public const SCENARIOS = ['carriers', 'pipeline', 'pipeline-loss', 'fallback', 'capacity', + 'carrier-loss-activity', 'carrier-loss-workflow', 'carrier-loss-workflow-completion', 'completion-transit', + 'carrier-expiry', 'carrier-refence', 'pipeline-buffer-age', 'prefetch-loss', 'default-off', 'explicit-off', 'packet-defaults', 'opposite-slots']; /** @param array $data */ private function __construct(public readonly array $data, public readonly string $vendor, public readonly string $sha256) diff --git a/scripts/conformance/Support/CompletionPollTrace.php b/scripts/conformance/Support/CompletionPollTrace.php index 9dac8acb..49c6af6d 100644 --- a/scripts/conformance/Support/CompletionPollTrace.php +++ b/scripts/conformance/Support/CompletionPollTrace.php @@ -6,6 +6,8 @@ use RuntimeException; +require_once __DIR__.'/CompletionPollFaults.php'; + /** Bounded synthetic wire metadata only. Credential headers and payloads are never retained. */ final class CompletionPollTrace { @@ -69,10 +71,16 @@ public function change(callable $operation): array */ public static function task(?array $task): ?array { - return $task === null ? null : array_intersect_key($task, array_flip([ + if ($task === null) return null; + $result = array_intersect_key($task, array_flip([ 'task_id', 'task_kind', 'workflow_id', 'run_id', 'lease_owner', 'workflow_task_attempt', - 'activity_attempt_id', 'lease_expires_at', + 'activity_attempt_id', 'attempt_number', 'lease_expires_at', ])); + if (is_string($task['task_id'] ?? null) && is_string($task['lease_owner'] ?? null) + && is_string($task['lease_expires_at'] ?? null)) { + $result['identity_sha256'] = CompletionPollFaults::identity(['task' => $task]); + } + return $result; } /** @param array $body @@ -98,10 +106,16 @@ public static function request(array $body, ?string $wire = null): array */ public static function answer(array $body): array { - $result = array_intersect_key($body, array_flip(['poll_status', 'reason', 'server_time', 'protocol_version'])); + $result = array_intersect_key($body, array_flip(['poll_status', 'reason', 'server_time', 'protocol_version', + 'task_id', 'lease_owner', 'workflow_task_attempt', 'activity_attempt_id', 'lease_expires_at', + 'renewed', 'recorded', 'outcome', 'worker_id', 'task_status', 'run_status', 'can_continue', 'current_attempt'])); + if (is_string($body['registration_token'] ?? null) && preg_match('/\A[a-f0-9]{32}\z/', $body['registration_token']) === 1) { + $result['registration_sha256'] = hash('sha256', $body['registration_token']); + } $result['task'] = self::task(is_array($body['task'] ?? null) ? $body['task'] : null); $result['capabilities'] = array_intersect_key($body['server_capabilities'] ?? [], array_flip([ 'workflow_task_completion_activity_poll', 'activity_completion_workflow_poll', 'workflow_task_completion_workflow_poll', + 'worker_deregistration_fencing', ])); foreach (['activity_poll', 'workflow_poll'] as $key) { if (!array_key_exists($key, $body)) { @@ -109,6 +123,7 @@ public static function answer(array $body): array } $answer = $body[$key]['answer'] ?? null; $result[$key] = ['leased' => $body[$key]['leased'] ?? null, + 'poll_status' => $answer['poll_status'] ?? null, 'server_time' => $answer['server_time'] ?? null, 'task' => self::task(is_array($answer['task'] ?? null) ? $answer['task'] : null)]; } return $result; diff --git a/scripts/conformance/Support/completion-poll-relay.php b/scripts/conformance/Support/completion-poll-relay.php index 0ba1562d..036e0fc5 100644 --- a/scripts/conformance/Support/completion-poll-relay.php +++ b/scripts/conformance/Support/completion-poll-relay.php @@ -5,9 +5,11 @@ // A loopback test relay forwards the actual runtime answer; it never implements a Server response. require_once __DIR__.'/CompletionPollSubject.php'; require_once __DIR__.'/CompletionPollTrace.php'; +require_once __DIR__.'/CompletionPollRelayFaults.php'; use DurableWorkflow\Qualification\CompletionPollSubject; use DurableWorkflow\Qualification\CompletionPollTrace; +use DurableWorkflow\Qualification\CompletionPollRelayFaults; try { $trace = new CompletionPollTrace((string) getenv('PIPELINE_TEST_STATE')); @@ -32,6 +34,7 @@ $trace->record(['kind' => 'request', 'request' => $requestId, 'path' => $tracePath, 'body' => CompletionPollTrace::request($body, $raw)]); } + CompletionPollRelayFaults::before($trace, $path, $body, $requestId); $headers = []; foreach (getallheaders() as $key => $value) { if (!in_array(strtolower($key), ['host', 'content-length', 'connection', 'transfer-encoding'], true)) { @@ -67,8 +70,13 @@ $trace->record(['kind' => 'upstream', 'request' => $requestId, 'status' => $status, 'answer' => is_array($answer) ? CompletionPollTrace::answer($answer) : null]); } + if (is_array($answer) && CompletionPollRelayFaults::after($trace, $path, $answer, $status, $requestId)) { + $status = 502; + $response = 'The test relay deliberately withheld a successful upstream answer.'; + $contentType = 'text/plain'; + } $parent = $trace->change(static function (array &$state) use ($path, $status, $requestId): void { - if (!in_array($state['scenario'], ['pipeline', 'pipeline-loss'], true) + if (!in_array($state['scenario'], ['pipeline', 'pipeline-loss', 'pipeline-buffer-age', 'default-off', 'explicit-off'], true) || !preg_match('~/workflow-tasks/([^/]+)/complete$~', $path, $match)) { return; } diff --git a/scripts/conformance/completion-polls.md b/scripts/conformance/completion-polls.md index fb4fbdd9..3be7853b 100644 --- a/scripts/conformance/completion-polls.md +++ b/scripts/conformance/completion-polls.md @@ -52,7 +52,7 @@ responses. It records bounded task/attempt/owner/expiry metadata, carrier IDs, callback order and hashes of actual request bytes. The relay implements no Server endpoint or durable transition. An upstream error stays an error. -The initial scenarios are: +The basic scenarios are: - `carriers`: three remote-activity workflows must exercise all three carried paths with inline lease answers and actual callbacks. Polling those same @@ -69,10 +69,9 @@ The initial scenarios are: producer lacks support. The controlled answer delay and loss are identified explicitly. They are not -an undelayed load comparison or a speedup measurement. Actual expiry/refencing, -additional opposite-slot/refusal controls and honest undelayed backend/load -comparisons remain required in the selected published-artifact acceptance. -The initial driver is not the complete #328–#332 release matrix. +an undelayed load comparison or a speedup measurement. The additional controls below are connected at Source scope. Undelayed load +support is still being prepared. Every selected row needs its own actual +published-artifact result before the #328–#332 matrix is complete. Each invocation has a selected 15–120 second product deadline, bounded HTTP requests and a 15 second Worker shutdown grace within a 30 second cleanup @@ -95,8 +94,9 @@ container inspection and runner-file binding alongside the result. The program's `pass` covers only its declared installed-consumer scenario. Keep the runner, `Support/CompletionPoll*.php`, the relay and both existing -process helpers, `tests/Support/CompletionPipelineScenario.php` and -`tests/Support/completion-pipeline-server.php`, in their relative paths. +process helpers in their relative paths: `tests/Support/CompletionPipelineScenario.php`, +`tests/Support/completion-pipeline-server.php`, `tests/Support/WorkerPipelineScenario.php`, +`tests/Support/completion-pipeline-http.php` and `tests/Support/RecordingMetricsRecorder.php`. Repository PHPStan includes these new source files. The pure subject and evidence unit tests use synthetic metadata only. Relay controls forward fixed HTTP data to test byte/role forwarding, error preservation and one @@ -104,24 +104,30 @@ successful-answer loss. They create no Server, package installation, profile or published-product acceptance. The remaining matrix is being prepared against the same accepted schema63 -API. The initial four scenario names remain the only executable selections -until their successors are connected and qualified. Delay selection uses the +API. All controls except the pending load cells have executable selections. +They remain unexecuted against an installed package and Native runtime. Delay selection uses the actual normal-poll `server_time` and original `lease_expires_at`. An absent clock, unsuitable grant window or insufficient caller budget blocks that control rather than substituting a worker clock or changing Server state. | Control | Actual operation and required observation | | --- | --- | -| Carried-answer loss, each of three paths | Withhold one actual successful completion answer. Recover its original poll ID and complete task/history identity. | -| Original completion transit | Delay the real completion request before its carried claim. Require original-request charging and confirmation before callback when the reported reserve is insufficient. | -| Expired carried lease | Hold the real answer past its reported original expiry. Require fenced confirmation/refusal and no callback or publication under that expired generation. | -| Registration replacement | Re-register this invocation's same synthetic worker through the public worker API. Retain only token digests and require old-grant refusal; clean the owned replacement with its actual fenced receipt. | -| Buffered lease age | Delay a real registration-heartbeat reply after prefetch. Require workflow lease confirmation before the aged task executes. Registration heartbeat does not renew task authority. | -| Opposite slot and missing result | Hold actual opposite-kind leases through public polls/completions. Observe real carrier omission, then reservation release and later work without disabling the capability. | -| Lost ordinary prefetch | Withhold one real successful prefetch answer. Preserve the original ID, backoff and lease/history on recovery without another reservation. | -| Default OFF | Exercise omitted and explicit-false options against the actual runtime. Keep the installed fixed-socket packet regression separate from native producer acceptance. | +| `carrier-loss-activity`, `carrier-loss-workflow`, `carrier-loss-workflow-completion` | Withhold one actual successful completion answer. Recover its original poll ID and complete task/history identity. | +| `completion-transit` | Delay the real completion request before its carried claim. Require original-request charging and confirmation before callback when the reported reserve is insufficient. | +| `carrier-expiry` | Hold the real answer past its reported original expiry. Require fenced confirmation/refusal and no callback or publication under that expired generation. | +| `carrier-refence` | Re-register this invocation's same synthetic worker through the public worker API. Retain only token digests and require original activity-poll recovery to return the actual `poll_cached_task_expired` refusal; clean the owned replacement with its actual fenced receipt. | +| `pipeline-buffer-age` | Delay a real registration-heartbeat reply after prefetch. Require workflow lease confirmation before the aged task executes. Registration heartbeat does not renew task authority. | +| `opposite-slots`, `capacity` | Hold actual opposite-kind leases through public polls/completions. Observe real carrier omission, then reservation release and later work without disabling the capability. | +| `prefetch-loss` | Withhold one real successful prefetch answer. Preserve the original ID, backoff and lease/history on recovery without another reservation. | +| `default-off`, `explicit-off`, `packet-defaults` | Exercise omitted and explicit-false options against the actual runtime. Keep the installed fixed-socket packet regression separate from native producer acceptance. | | Bounded load | Direct runtime calls with no relay delay or per-request trace. Compare explicit OFF/ON cells for six workers, 200 workflows and five 20ms local or remote steps. Retain all three repetitions and regressions. | +The registration control does not assert that replacement revokes an already +granted workflow lease. That distinct producer vector is requested in Server +#1067. `packet-defaults` runs the installed SDK against the fixed HTTP fixture +only and explicitly reports `runtime_observed: false`. `opposite-slots` uses +public Client polls/completions rather than managed Worker capacity. + Load cells require caller evidence for fresh isolated databases and the actual backend, image, host and resource settings. The runner does not create a database, invent an administrative reset or infer backend identity from a diff --git a/scripts/conformance/qualify-completion-polls.php b/scripts/conformance/qualify-completion-polls.php index 5461779a..b36dc6c3 100644 --- a/scripts/conformance/qualify-completion-polls.php +++ b/scripts/conformance/qualify-completion-polls.php @@ -3,8 +3,11 @@ declare(strict_types=1); use DurableWorkflow\Client; +use DurableWorkflow\Exception\ServerException; use DurableWorkflow\Qualification\CompletionPollEvidence; use DurableWorkflow\Qualification\CompletionPollFailure; +use DurableWorkflow\Qualification\CompletionPollPacketDefaults; +use DurableWorkflow\Qualification\CompletionPollOppositeSlots; use DurableWorkflow\Qualification\CompletionPollSubject; use DurableWorkflow\Qualification\CompletionPollTrace; use DurableWorkflow\Tests\Support\CompletionPipelineScenario; @@ -56,15 +59,20 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, $prefix = 'completion-'.bin2hex(random_bytes(8)); $workerId = $prefix.'-worker'; $queue = $prefix.'-queue'; - $pipeline = in_array($data['scenario'], ['pipeline', 'pipeline-loss'], true); + $pipeline = in_array($data['scenario'], ['pipeline', 'pipeline-loss', 'pipeline-buffer-age', 'prefetch-loss'], true); + $localWorkload = $pipeline || in_array($data['scenario'], ['default-off', 'explicit-off'], true); + $claimRefusal = in_array($data['scenario'], ['carrier-expiry', 'carrier-refence'], true); + $slow = in_array($data['scenario'], ['completion-transit', 'carrier-expiry', 'carrier-refence'], true); + $httpTimeout = $slow ? $data['deadline_seconds'] : 5; file_put_contents($stateFile, json_encode(['events' => [], 'endpoint' => $data['endpoint'], - 'prefix' => $prefix, 'scenario' => $data['scenario']], JSON_THROW_ON_ERROR)); + 'prefix' => $prefix, 'scenario' => $data['scenario'], 'deadline_at' => hrtime(true) / 1e9 + $data['deadline_seconds']], JSON_THROW_ON_ERROR)); $trace = new CompletionPollTrace($stateFile); - $control = new Client($relay, namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 5); + $control = new Client($relay, namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 2); if ($control->clusterInfo()->version !== $data['server_version']) { throw new RuntimeException('completion_runtime_version'); } $cleanup = new Client($data['endpoint'], namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 1); + $actor = new Client($data['endpoint'], namespace: $data['namespace'], workerToken: $workerToken, requestTimeoutSeconds: 2); $deadline = microtime(true) + $data['deadline_seconds']; $pid = pcntl_fork(); if ($pid === -1) { @@ -76,12 +84,27 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, throw new RuntimeException('completion_worker_process_group'); } $arguments = ['workerId' => $workerId]; - if ($pipeline) { - $arguments['enableCompletionPipelining'] = true; + if ($pipeline || $data['scenario'] === 'explicit-off') { + $arguments['enableCompletionPipelining'] = $pipeline; } + if (in_array($data['scenario'], ['default-off', 'explicit-off'], true)) { + foreach (['__construct', 'create'] as $method) { + $options = array_filter((new ReflectionMethod(Worker::class, $method))->getParameters(), + static fn ($parameter): bool => $parameter->getName() === 'enableCompletionPipelining'); + if (count($options) !== 1 || current($options)->getDefaultValue() !== false) { + throw CompletionPollFailure::product('completion_default_option_not_false'); + } + } + $trace->record(['kind' => 'default_option', 'default' => false, 'explicit' => $data['scenario'] === 'explicit-off']); + } + if ($data['scenario'] === 'pipeline-buffer-age') $arguments['heartbeatIntervalSeconds'] = 1; + $arguments['transientPollRetryObserver'] = static function (string $kind, int $attempt, float $delay, ServerException $error) use ($trace): void { + $trace->record(['kind' => 'poll_retry', 'task_kind' => $kind, 'attempt' => $attempt, 'delay' => $delay, + 'status' => $error->status, 'reason' => $error->reason]); + }; $worker = new Worker(new Client($relay, namespace: $data['namespace'], workerToken: $workerToken, - requestTimeoutSeconds: 5, workerRequestTimeoutSeconds: 5), $queue, ...$arguments); - $worker->registerWorkflow('qualification.completion', static function (WorkflowContext $context, mixed ...$arguments) use ($trace, $prefix, $pipeline): mixed { + requestTimeoutSeconds: 5, workerRequestTimeoutSeconds: $httpTimeout), $queue, ...$arguments); + $worker->registerWorkflow('qualification.completion', static function (WorkflowContext $context, mixed ...$arguments) use ($trace, $prefix, $pipeline, $localWorkload): mixed { $index = completionIndex($arguments); if ($pipeline && $index === 3) { $until = microtime(true) + 1; @@ -94,7 +117,7 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, } $trace->record(['kind' => 'callback', 'index' => $index, 'workflow_id' => $context->workflowId, 'run_id' => $context->runId]); - if ($pipeline) { + if ($localWorkload) { if ($index > 1) { for ($step = 0; $step < 5; ++$step) { $context->localActivity('qualification.local', [$index]); @@ -109,7 +132,8 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, $trace->record(['kind' => 'activity', 'task_id' => $context->taskId, 'index' => $index]); return $index; }); - $worker->registerActivity('qualification.local', static function (ActivityContext $context, mixed ...$arguments) use ($trace): int { + $localCalls = []; + $worker->registerActivity('qualification.local', static function (ActivityContext $context, mixed ...$arguments) use ($trace, $data, &$localCalls): int { $index = completionIndex($arguments); $trace->record(['kind' => 'local', 'index' => $index]); if ($index === 3) { @@ -117,35 +141,87 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, $state['release_parent'] = true; }); } - usleep(20_000); + $localCalls[$index] = ($localCalls[$index] ?? 0) + 1; + usleep($data['scenario'] === 'pipeline-buffer-age' && $index === 2 && $localCalls[$index] === 5 ? 1_250_000 : 20_000); return $index; }); + // Queue the complete controlled workload before the worker + // starts, so eager carriers have real eligible successors. + while (!(($trace->state())['workload_ready'] ?? false)) { + if (microtime(true) >= $deadline) throw new RuntimeException('completion_workload_start_deadline'); + usleep(10_000); + } $worker->run(1); exit(0); } catch (Throwable $failure) { // Headers, payloads and arbitrary runtime exception messages stay out of diagnostics. - $trace->record(['kind' => 'worker_error', 'class' => $failure::class]); + $trace->record(['kind' => 'worker_error', 'class' => $failure::class, + 'status' => $failure instanceof ServerException ? $failure->status : null, + 'reason' => $failure instanceof ServerException ? $failure->reason : null]); exit(1); } } $handles = []; + for ($index = 1; $index <= ($localWorkload ? 5 : 3); ++$index) { + // Retain every owned ID before a start can have an unknown result. + $handles[$index] = $control->workflowHandle($prefix.'-'.$index); + } $failure = null; + $expectedStop = false; + $replacementToken = null; try { - while (($control->listWorkers($queue)['workers'] ?? []) === []) { - if (microtime(true) >= $deadline) { - throw new RuntimeException('completion_registration_deadline'); - } - usleep(50_000); - } - for ($index = 1; $index <= ($pipeline ? 5 : 3); ++$index) { + for ($index = 1; $index <= ($localWorkload ? 5 : 3); ++$index) { $handles[$index] = $control->startWorkflow('qualification.completion', $prefix.'-'.$index, $queue, input: [$index], executionTimeoutSeconds: 180, runTimeoutSeconds: 150); } - foreach ($handles as $index => $handle) { - $remaining = (int) floor($deadline - microtime(true)); - if ($remaining < 1 || $handle->result(timeoutSeconds: $remaining, pollIntervalSeconds: 0.1) !== $index) { - throw CompletionPollFailure::product('completion_workflow_result'); + $trace->change(static function (array &$value): void { $value['workload_ready'] = true; }); + while (($control->listWorkers($queue)['workers'] ?? []) === []) { + if (microtime(true) >= $deadline) throw new RuntimeException('completion_registration_deadline'); + usleep(50_000); + } + $pending = $handles; + while ($pending !== []) { + if (microtime(true) >= $deadline) throw new RuntimeException('completion_workflow_deadline'); + $state = $trace->state(); + if (($state['refence_ready'] ?? false) && $replacementToken === null) { + $old = array_values(array_filter($state['events'], static fn (array $event): bool => + $event['kind'] === 'upstream' && isset($event['answer']['registration_sha256']))); + if ($old === []) throw new RuntimeException('completion_refence_receipt_required'); + $replacement = $actor->registerWorker($workerId, $queue, ['qualification.completion'], + ['qualification.remote', 'qualification.local']); + $replacementToken = $replacement['registration_token'] ?? null; + if (($replacement['registered'] ?? null) !== true || ($replacement['worker_id'] ?? null) !== $workerId + || ($replacement['task_queue'] ?? null) !== $queue + || !is_string($replacementToken) || preg_match('/\A[a-f0-9]{32}\z/', $replacementToken) !== 1) { + throw new RuntimeException('completion_refence_receipt_required'); + } + $digest = hash('sha256', $replacementToken); + if ($digest === $old[0]['answer']['registration_sha256']) { + throw CompletionPollFailure::product('completion_refence_token_not_rotated'); + } + $trace->record(['kind' => 'refence', 'worker_id' => $workerId, + 'old_registration_sha256' => $old[0]['answer']['registration_sha256'], 'registration_sha256' => $digest]); + $trace->change(static function (array &$value): void { $value['refence_complete'] = true; }); } + if (array_filter($state['events'], static fn (array $event): bool => $event['kind'] === 'relay_error') !== []) { + throw new RuntimeException('completion_relay_failed'); + } + if ($claimRefusal && array_filter($state['events'], static fn (array $event): bool => $event['kind'] === 'worker_error') !== []) { + $expectedStop = true; + break; + } + foreach ($pending as $index => $handle) { + $description = $handle->describeSelectedRun(); + if ($description->status === 'completed') { + if ($handle->resultOfSelectedRun(timeoutSeconds: 1, pollIntervalSeconds: 0.1) !== $index) { + throw CompletionPollFailure::product('completion_workflow_result'); + } + unset($pending[$index]); + } elseif (in_array($description->status, ['failed', 'cancelled', 'terminated', 'timed_out'], true)) { + throw CompletionPollFailure::product('completion_workflow_result'); + } + } + if ($pending !== []) usleep(50_000); } } catch (Throwable $error) { $failure = $error; @@ -167,14 +243,16 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, } usleep(50_000); } - if ($waited === -1 || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { - $failure ??= new RuntimeException('completion_worker_failed'); + if ($waited === -1) { + $failure = new RuntimeException('completion_worker_reap_failed', previous: $failure); + } elseif (!pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { + if (!$expectedStop) $failure ??= new RuntimeException('completion_worker_failed'); } if (posix_kill(-$pid, 0)) { posix_kill(-$pid, SIGKILL); $failure = new RuntimeException('completion_worker_helper_survived', previous: $failure); } - if ($failure !== null) { + if ($failure !== null || $expectedStop) { foreach ($handles as $handle) { if (microtime(true) >= $cleanupDeadline - 2) { $trace->record(['kind' => 'cleanup_error']); @@ -187,21 +265,36 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, $cleanup->terminateWorkflow($handle->workflowId, 'Completion qualification cleanup', $handle->selectedRunId); } } catch (Throwable $cleanupError) { + if ($cleanupError instanceof ServerException && $cleanupError->status === 404 && $handle->selectedRunId === null) continue; $trace->record(['kind' => 'cleanup_error']); $failure = new RuntimeException('completion_workflow_cleanup_failed', previous: $cleanupError); } } } + if (is_string($replacementToken)) { + try { + $actor->deregisterWorkerRegistration($workerId, $replacementToken); + } catch (Throwable $error) { + $failure = new RuntimeException('completion_refence_cleanup_failed', previous: $error); + } + } } if ($failure !== null) { - throw $failure; + return ['scenario' => $data['scenario'], 'outcome' => CompletionPollFailure::diagnostic($failure)['outcome'], + 'diagnostic' => CompletionPollFailure::diagnostic($failure), 'events' => $trace->state()['events']]; } if (($cleanup->listWorkers($queue)['workers'] ?? []) !== []) { throw new RuntimeException('completion_worker_registration_survived'); } $events = $trace->state()['events']; - return ['scenario' => $data['scenario'], 'assertions' => CompletionPollEvidence::verify($events, $data['scenario'], $workerId, $prefix), - 'workflow_count' => count($handles), 'worker_deregistered' => true, 'events' => $events]; + try { + $assertions = CompletionPollEvidence::verify($events, $data['scenario'], $workerId, $prefix); + return ['scenario' => $data['scenario'], 'outcome' => 'pass', 'assertions' => $assertions, + 'workflow_count' => count($handles), 'worker_deregistered' => true, 'expected_claim_refusal_stop' => $expectedStop, 'events' => $events]; + } catch (Throwable $error) { + return ['scenario' => $data['scenario'], 'outcome' => CompletionPollFailure::diagnostic($error)['outcome'], + 'diagnostic' => CompletionPollFailure::diagnostic($error), 'events' => $events]; + } }, __DIR__.'/Support/completion-poll-relay.php', true); } @@ -216,16 +309,29 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, if (!function_exists('pcntl_fork') || !function_exists('posix_kill') || !extension_loaded('curl')) { throw new RuntimeException('completion_toolchain'); } + if ($subject->data['scenario'] === 'packet-defaults') { + require_once __DIR__.'/Support/CompletionPollPacketDefaults.php'; + echo json_encode(['schema' => 'durable-workflow.php-completion-polls-result/v1', 'outcome' => 'pass', + 'identity' => $identity, 'result' => CompletionPollPacketDefaults::run()], JSON_THROW_ON_ERROR).PHP_EOL; + exit(0); + } $clientToken = getenv('COMPLETION_TEST_CLIENT_TOKEN'); $workerToken = getenv('COMPLETION_TEST_WORKER_TOKEN'); if (!is_string($clientToken) || $clientToken === '' || !is_string($workerToken) || $workerToken === '' || $clientToken === $workerToken) { throw new RuntimeException('completion_scoped_test_tokens'); } - $result = completionScenario($subject, $clientToken, $workerToken); - echo json_encode(['schema' => 'durable-workflow.php-completion-polls-result/v1', 'outcome' => 'pass', + if ($subject->data['scenario'] === 'opposite-slots') { + require_once __DIR__.'/Support/CompletionPollOppositeSlots.php'; + $result = CompletionPollOppositeSlots::run($subject, $clientToken, $workerToken); + } else { + $result = completionScenario($subject, $clientToken, $workerToken); + } + $outcome = $result['outcome']; + echo json_encode(['schema' => 'durable-workflow.php-completion-polls-result/v1', 'outcome' => $outcome, 'scope' => 'installed SDK on caller-owned loopback runtime; image publication/digest and execution admission are caller evidence', 'identity' => $identity, 'server_version' => $subject->data['server_version'], 'caller_server_image' => $subject->data['server_image'], 'result' => $result], JSON_THROW_ON_ERROR).PHP_EOL; + exit($outcome === 'pass' ? 0 : 1); } catch (Throwable $error) { fwrite(STDERR, json_encode(CompletionPollFailure::diagnostic($error), JSON_THROW_ON_ERROR).PHP_EOL); exit(1); diff --git a/tests/CompletionPollConsumerFaultsTest.php b/tests/CompletionPollConsumerFaultsTest.php index 2d522b72..15335cff 100644 --- a/tests/CompletionPollConsumerFaultsTest.php +++ b/tests/CompletionPollConsumerFaultsTest.php @@ -14,7 +14,6 @@ final class CompletionPollConsumerFaultsTest extends TestCase public function testTransitAndExpiryAreSelectedFromTheOriginalProducerWindow(): void { self::assertSame(22.0, CompletionPollFaults::delay(self::grant(), 'transit', 60)); - self::assertSame(22.0, CompletionPollFaults::delay(self::grant(), 'refence', 60)); self::assertSame(30.25, CompletionPollFaults::delay(self::grant(), 'expiry', 60)); } diff --git a/tests/CompletionPollConsumerMatrixEvidenceTest.php b/tests/CompletionPollConsumerMatrixEvidenceTest.php new file mode 100644 index 00000000..7bcb0071 --- /dev/null +++ b/tests/CompletionPollConsumerMatrixEvidenceTest.php @@ -0,0 +1,329 @@ +expectExceptionMessage('completion_evidence_loss_not_targeted'); + CompletionPollEvidence::verify(self::loss('activity_poll', 'workflow'), 'carrier-loss-workflow', 'worker', 'fixture'); + } + + public function testRecoveryMustKeepOriginalIdAndImmutableDeadline(): void + { + $events = self::loss('workflow_poll', 'activity'); + $events[7]['body']['poll_request_id'] = 'fresh-id'; + $this->expectExceptionMessage('completion_evidence_recovery_identity_changed'); + CompletionPollEvidence::verify($events, 'carrier-loss-workflow', 'worker', 'fixture'); + } + + public function testRestampingTheRecoveredLeaseCannotPass(): void + { + $events = self::loss('workflow_poll', 'activity'); + $changed = self::task('workflow'); + $changed['lease_expires_at'] = '2026-11-07T08:01:00.000000Z'; + $events[8]['answer'] = CompletionPollTrace::answer(self::grant($changed)); + $this->expectExceptionMessage('completion_evidence_replay_authority_changed'); + CompletionPollEvidence::verify($events, 'carrier-loss-workflow', 'worker', 'fixture'); + } + + public function testRetryingDifferentCompletionBytesCannotPass(): void + { + $events = self::loss('workflow_poll', 'activity'); + $events[4]['body']['body_sha256'] = str_repeat('c', 64); + $this->expectExceptionMessage('completion_evidence_completion_retry_changed'); + CompletionPollEvidence::verify($events, 'carrier-loss-workflow', 'worker', 'fixture'); + } + + public function testReplayedMetadataWithoutACallbackDoesNotQualifyDelivery(): void + { + $events = self::loss('workflow_poll', 'activity'); + array_pop($events); + $this->expectExceptionMessage('completion_evidence_recovered_callback_missing'); + CompletionPollEvidence::verify($events, 'carrier-loss-workflow', 'worker', 'fixture'); + } + + public function testCompletionTransitNeedsIdentityBoundConfirmationBeforeCallback(): void + { + $events = self::transit(); + $result = CompletionPollEvidence::verify($events, 'completion-transit', 'worker', 'fixture'); + self::assertTrue($result['confirmed_before_callback']); + $events[6]['answer']['workflow_task_attempt'] = 2; + $this->expectExceptionMessage('completion_evidence_confirmation_missing'); + CompletionPollEvidence::verify($events, 'completion-transit', 'worker', 'fixture'); + } + + public function testAShortRequestCannotQualifyWholeCompletionTransitCharging(): void + { + $events = self::transit(); + $events[4]['at'] = 2_000_000_000; + $this->expectExceptionMessage('completion_evidence_completion_transit_not_targeted'); + CompletionPollEvidence::verify($events, 'completion-transit', 'worker', 'fixture'); + } + + public function testRealExpiryRefusalNeedsNoCallbackOrPublicationForTheOldGeneration(): void + { + $events = self::expiry(); + self::assertTrue(CompletionPollEvidence::verify($events, 'carrier-expiry', 'worker', 'fixture')['original_expiry_reached']); + $events[] = self::event('callback', 32, ['workflow_id' => 'fixture-3']); + $this->expectExceptionMessage('completion_evidence_expired_callback'); + CompletionPollEvidence::verify($events, 'carrier-expiry', 'worker', 'fixture'); + } + + public function testAnUnrelatedServerFailureIsNotAnExpiryRefusal(): void + { + $events = self::expiry(); + $events[5]['status'] = 500; + $events[5]['answer']['reason'] = 'internal_error'; + $this->expectExceptionMessage('completion_evidence_expiry_refusal_missing'); + CompletionPollEvidence::verify($events, 'carrier-expiry', 'worker', 'fixture'); + } + + public function testActivityRefenceRequiresRotationAndRealOriginalIdRefusal(): void + { + $events = self::loss('activity_poll', 'workflow'); + $events[2]['mode'] = 'refenced_activity_answer'; + $events[8]['status'] = 409; + $events[8]['answer'] = ['reason' => 'poll_cached_task_expired']; + $events[9]['status'] = 409; + array_pop($events); + array_splice($events, 2, 0, [self::event('refence', 0.15, ['worker_id' => 'worker', + 'old_registration_sha256' => str_repeat('a', 64), 'registration_sha256' => str_repeat('b', 64)])]); + $result = CompletionPollEvidence::verify($events, 'carrier-refence', 'worker', 'fixture'); + self::assertTrue($result['original_activity_recovery_refused']); + self::assertFalse($result['existing_workflow_grant_revocation_asserted']); + $events[2]['registration_sha256'] = $events[2]['old_registration_sha256']; + $this->expectExceptionMessage('completion_evidence_refence_recovery'); + CompletionPollEvidence::verify($events, 'carrier-refence', 'worker', 'fixture'); + } + + public function testBufferedAgeNeedsARealDelayAndLaterLeaseConfirmation(): void + { + $events = self::buffer(); + self::assertTrue(CompletionPollEvidence::verify($events, 'pipeline-buffer-age', 'worker', 'fixture')['confirmed_before_callback']); + $events = array_values(array_filter($events, static fn (array $event): bool => ($event['request'] ?? null) !== 'confirm')); + $this->expectExceptionMessage('completion_evidence_confirmation_missing'); + CompletionPollEvidence::verify($events, 'pipeline-buffer-age', 'worker', 'fixture'); + } + + public function testDefaultOffRequiresBothTheActualDefaultAndSynchronousDelivery(): void + { + $events = self::buffer(); + array_unshift($events, self::event('default_option', 0, ['default' => false, 'explicit' => false])); + // Move the parent delivery ahead of the callback for the synchronous control. + $parent = array_values(array_filter($events, static fn (array $event): bool => $event['kind'] === 'delivered' && $event['request'] === 'parent'))[0]; + $parent['at'] = 1_850_000_000; + $events = array_values(array_filter($events, static fn (array $event): bool => !($event['kind'] === 'delivered' && $event['request'] === 'parent'))); + $at = array_keys(array_filter($events, static fn (array $event): bool => $event['kind'] === 'callback'))[0]; + array_splice($events, $at, 0, [$parent]); + self::assertTrue(CompletionPollEvidence::verify($events, 'default-off', 'worker', 'fixture')['completion_remained_synchronous']); + $events[0]['default'] = true; + $this->expectExceptionMessage('completion_evidence_default_option'); + CompletionPollEvidence::verify($events, 'default-off', 'worker', 'fixture'); + } + + public function testMissingResultReleasesCapacityWithoutDisablingTheCapability(): void + { + $events = self::capacity(); + self::assertTrue(CompletionPollEvidence::verify($events, 'capacity', 'worker', 'fixture')['missing_key_released_reservation']); + $events[1]['answer']['workflow_poll'] = ['leased' => false]; + $this->expectExceptionMessage('completion_evidence_missing_key_capacity'); + CompletionPollEvidence::verify($events, 'capacity', 'worker', 'fixture'); + } + + public function testLostPrefetchKeepsBackoffAndPublishesParentBeforeOriginalRecovery(): void + { + $events = self::prefetch(); + self::assertTrue(CompletionPollEvidence::verify($events, 'prefetch-loss', 'worker', 'fixture')['original_poll_id_recovered']); + $events = array_values(array_filter($events, static fn (array $event): bool => $event['kind'] !== 'poll_retry')); + $this->expectExceptionMessage('completion_evidence_prefetch_recovery_policy'); + CompletionPollEvidence::verify($events, 'prefetch-loss', 'worker', 'fixture'); + } + + private static function capacity(): array + { + $events = []; + foreach ([['workflow', 'activity_poll', 'activity'], ['activity', 'workflow_poll', 'workflow'], + ['workflow', 'workflow_poll', 'workflow']] as $index => [$parent, $member, $kind]) { + $task = array_replace(self::task($kind), ['task_id' => 'task-'.$index, 'workflow_id' => 'fixture-'.$index]); + $body = ['lease_owner' => 'worker', 'workflow_task_attempt' => 1, + $member => ['worker_id' => 'worker', 'poll_request_id' => 'carrier-'.$index]]; + if ($index === 0) $body['workflow_poll'] = ['worker_id' => 'worker', 'poll_request_id' => 'omitted']; + array_push($events, + self::event('request', $index, ['request' => 'parent-'.$index, 'path' => '/api/worker/'.$parent.'-tasks/parent/complete', + 'body' => CompletionPollTrace::request($body)]), + self::event('upstream', $index + 0.1, ['request' => 'parent-'.$index, 'status' => 200, + 'answer' => CompletionPollTrace::answer(['server_capabilities' => array_fill_keys(CompletionPollEvidence::CAPABILITIES, true), + $member => ['leased' => true, 'answer' => self::grant($task)]])]), + self::event('delivered', $index + 0.2, ['request' => 'parent-'.$index, 'status' => 200]), + self::event($kind === 'workflow' ? 'callback' : 'activity', $index + 0.3, + $kind === 'workflow' ? ['workflow_id' => $task['workflow_id']] : ['task_id' => $task['task_id']])); + } + return $events; + } + + private static function prefetch(): array + { + $parent = array_replace(self::task('workflow'), ['task_id' => 'task-2', 'workflow_id' => 'fixture-2']); + return [ + self::event('request', 0, ['request' => 'first', 'path' => '/api/worker/workflow-tasks/poll', 'body' => []]), + self::event('upstream', 0.01, ['request' => 'first', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant($parent))]), + self::event('delivered', 0.02, ['request' => 'first', 'status' => 200]), + self::event('request', 0.1, ['request' => 'prefetch', 'path' => '/api/worker/workflow-tasks/poll', + 'body' => ['worker_id' => 'worker', 'poll_request_id' => 'original-id']]), + self::event('upstream', 0.2, ['request' => 'prefetch', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant(self::task('workflow')))]), + self::event('fault', 0.21, ['request' => 'prefetch', 'mode' => 'lost_prefetch_answer']), + self::event('delivered', 0.3, ['request' => 'prefetch', 'status' => 502]), + self::event('request', 0.31, ['request' => 'parent', 'path' => '/api/worker/workflow-tasks/task-2/complete', 'body' => []]), + self::event('upstream', 0.32, ['request' => 'parent', 'status' => 200, 'answer' => ['recorded' => true]]), + self::event('delivered', 0.33, ['request' => 'parent', 'status' => 200]), + self::event('poll_retry', 0.34, ['task_kind' => 'workflow', 'attempt' => 1, 'delay' => 0.1]), + self::event('request', 0.45, ['request' => 'recover', 'path' => '/api/worker/workflow-tasks/poll', + 'body' => ['worker_id' => 'worker', 'poll_request_id' => 'original-id']]), + self::event('upstream', 0.5, ['request' => 'recover', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant(self::task('workflow')))]), + self::event('delivered', 0.6, ['request' => 'recover', 'status' => 200]), + self::event('callback', 0.7, ['workflow_id' => 'fixture-3']), + ]; + } + + /** Fixed observation vectors only; these implement no Server or installed acceptance. */ + private static function loss(string $member, string $parent): array + { + $kind = $member === 'activity_poll' ? 'activity' : 'workflow'; + $body = ['lease_owner' => 'worker', 'workflow_task_attempt' => 1, + $member => ['worker_id' => 'worker', 'poll_request_id' => 'original-id', 'timeout_seconds' => 0]]; + $path = '/api/worker/'.$parent.'-tasks/parent/complete'; + return [ + self::event('request', 0, ['request' => 'parent', 'path' => $path, 'body' => CompletionPollTrace::request($body)]), + self::event('upstream', 0.1, ['request' => 'parent', 'status' => 200, 'answer' => CompletionPollTrace::answer([ + $member => ['leased' => true, 'answer' => self::grant(self::task($kind))]])]), + self::event('fault', 0.2, ['request' => 'parent', 'mode' => 'lost_carried_answer', 'member' => $member]), + self::event('delivered', 0.3, ['request' => 'parent', 'status' => 502]), + self::event('request', 0.4, ['request' => 'retry', 'path' => $path, 'body' => CompletionPollTrace::request($body)]), + self::event('upstream', 0.5, ['request' => 'retry', 'status' => 200, 'answer' => ['recorded' => false, 'reason' => 'already_completed']]), + self::event('delivered', 0.6, ['request' => 'retry', 'status' => 200]), + self::event('request', 0.7, ['request' => 'recover', 'path' => '/api/worker/'.$kind.'-tasks/poll', + 'body' => ['worker_id' => 'worker', 'poll_request_id' => 'original-id']]), + self::event('upstream', 0.8, ['request' => 'recover', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant(self::task($kind)))]), + self::event('delivered', 0.9, ['request' => 'recover', 'status' => 200]), + self::event($kind === 'workflow' ? 'callback' : 'activity', 1, $kind === 'workflow' ? ['workflow_id' => 'fixture-3'] : ['task_id' => 'task-3']), + ]; + } + + private static function transit(): array + { + $task = self::task('workflow'); + return [ + self::event('request', 0, ['request' => 'parent', 'path' => '/api/worker/workflow-tasks/parent/complete', 'body' => []]), + self::event('fault_begin', 0.1, ['request' => 'parent', 'mode' => 'completion_transit']), + self::event('fault_end', 22.1, ['request' => 'parent', 'mode' => 'completion_transit']), + self::event('upstream', 22.2, ['request' => 'parent', 'status' => 200, + 'answer' => CompletionPollTrace::answer(['workflow_poll' => ['leased' => true, 'answer' => self::grant($task)]])]), + self::event('delivered', 22.3, ['request' => 'parent', 'status' => 200]), + self::event('request', 22.4, ['request' => 'confirm', 'path' => '/api/worker/workflow-tasks/task-3/heartbeat', + 'body' => ['lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('upstream', 22.5, ['request' => 'confirm', 'status' => 200, 'answer' => ['task_id' => 'task-3', + 'lease_owner' => 'worker', 'workflow_task_attempt' => 1, 'renewed' => true]]), + self::event('delivered', 22.6, ['request' => 'confirm', 'status' => 200]), + self::event('callback', 22.7, ['workflow_id' => 'fixture-3']), + ]; + } + + private static function expiry(): array + { + return [ + self::event('request', 0, ['request' => 'parent', 'path' => '/api/worker/activity-tasks/parent/complete', 'body' => []]), + self::event('upstream', 0.1, ['request' => 'parent', 'status' => 200, + 'answer' => CompletionPollTrace::answer(['workflow_poll' => ['leased' => true, 'answer' => self::grant(self::task('workflow'))]])]), + self::event('fault_begin', 0.2, ['request' => 'parent', 'mode' => 'expiry']), + self::event('delivered', 30.4, ['request' => 'parent', 'status' => 200]), + self::event('request', 30.5, ['request' => 'confirm', 'path' => '/api/worker/workflow-tasks/task-3/heartbeat', + 'body' => ['lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('upstream', 30.6, ['request' => 'confirm', 'status' => 409, + 'answer' => ['reason' => 'lease_expired', 'task_id' => 'task-3', 'workflow_task_attempt' => 1]]), + self::event('delivered', 30.7, ['request' => 'confirm', 'status' => 409]), + self::event('worker_error', 30.8, ['class' => 'DurableWorkflow\\Exception\\ServerException']), + ]; + } + + private static function task(string $kind): array + { + $task = ['task_id' => 'task-3', 'task_kind' => $kind, 'workflow_id' => 'fixture-3', 'run_id' => 'run-3', + 'lease_owner' => 'worker', 'lease_expires_at' => '2026-11-07T08:00:30.000000Z', 'history_events' => []]; + return $task + ($kind === 'workflow' ? ['workflow_task_attempt' => 1] : ['activity_attempt_id' => 'activity-attempt-1', 'attempt_number' => 1]); + } + + private static function grant(array $task): array + { + return ['task' => $task, 'poll_status' => 'leased', 'server_time' => '2026-11-07T08:00:00.000000Z', + 'server_capabilities' => array_fill_keys(CompletionPollEvidence::CAPABILITIES, true)]; + } + + private static function buffer(): array + { + $two = array_replace(self::task('workflow'), ['task_id' => 'task-2', 'workflow_id' => 'fixture-2']); + $three = self::task('workflow'); + return [ + self::event('request', 0, ['request' => 'original', 'path' => '/api/worker/workflow-tasks/poll', + 'body' => ['worker_id' => 'worker', 'timeout_seconds' => 0, 'poll_request_id' => 'original-2']]), + self::event('upstream', 0.01, ['request' => 'original', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant($two))]), + self::event('delivered', 0.02, ['request' => 'original', 'status' => 200]), + self::event('request', 0.1, ['request' => 'prefetch', 'path' => '/api/worker/workflow-tasks/poll', + 'body' => ['worker_id' => 'worker', 'timeout_seconds' => 0, 'poll_request_id' => 'prefetch-3']]), + self::event('upstream', 0.11, ['request' => 'prefetch', 'status' => 200, 'answer' => CompletionPollTrace::answer(self::grant($three))]), + self::event('delivered', 0.12, ['request' => 'prefetch', 'status' => 200]), + self::event('request', 0.2, ['request' => 'registration-heartbeat', 'path' => '/api/worker/heartbeat', 'body' => ['worker_id' => 'worker']]), + self::event('upstream', 0.21, ['request' => 'registration-heartbeat', 'status' => 200, 'answer' => []]), + self::event('fault_begin', 0.22, ['request' => 'registration-heartbeat', 'mode' => 'buffer_age']), + self::event('fault_end', 1.5, ['request' => 'registration-heartbeat', 'mode' => 'buffer_age']), + self::event('delivered', 1.51, ['request' => 'registration-heartbeat', 'status' => 200]), + self::event('request', 1.6, ['request' => 'parent', 'path' => '/api/worker/workflow-tasks/task-2/complete', + 'body' => ['lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('upstream', 1.7, ['request' => 'parent', 'status' => 200, 'answer' => []]), + self::event('request', 1.8, ['request' => 'confirm', 'path' => '/api/worker/workflow-tasks/task-3/heartbeat', + 'body' => ['lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('upstream', 1.81, ['request' => 'confirm', 'status' => 200, 'answer' => ['renewed' => true, + 'task_id' => 'task-3', 'lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('delivered', 1.82, ['request' => 'confirm', 'status' => 200]), + self::event('callback', 1.9, ['workflow_id' => 'fixture-3']), + self::event('local', 1.95, ['index' => 3]), + self::event('delivered', 2, ['request' => 'parent', 'status' => 200]), + self::event('request', 2.1, ['request' => 'child', 'path' => '/api/worker/workflow-tasks/task-3/complete', + 'body' => ['lease_owner' => 'worker', 'workflow_task_attempt' => 1]]), + self::event('upstream', 2.11, ['request' => 'child', 'status' => 200, 'answer' => []]), + self::event('delivered', 2.12, ['request' => 'child', 'status' => 200]), + ]; + } + + private static function event(string $kind, float $seconds, array $fields): array + { + return ['kind' => $kind, 'at' => (int) ($seconds * 1e9)] + $fields; + } +} diff --git a/tests/CompletionPollConsumerOppositeSlotsTest.php b/tests/CompletionPollConsumerOppositeSlotsTest.php new file mode 100644 index 00000000..1307844c --- /dev/null +++ b/tests/CompletionPollConsumerOppositeSlotsTest.php @@ -0,0 +1,34 @@ + true], 'workflow_poll'); + $this->addToAssertionCount(1); + } + + #[DataProvider('notDeclined')] + public function testAnEmptyClaimOrInvalidParentIsNotAnOppositeSlotDecline(array $answer): void + { + $this->expectExceptionMessage('completion_opposite_carrier_not_declined'); + CompletionPollOppositeSlots::omitted($answer, 'workflow_poll'); + } + + public static function notDeclined(): iterable + { + yield 'empty probe' => [['recorded' => true, 'workflow_poll' => ['leased' => false]]]; + yield 'extra lease' => [['recorded' => true, 'workflow_poll' => ['leased' => true]]]; + yield 'parent not recorded' => [['recorded' => false]]; + } +} diff --git a/tests/CompletionPollConsumerPacketDefaultsTest.php b/tests/CompletionPollConsumerPacketDefaultsTest.php new file mode 100644 index 00000000..15f88b80 --- /dev/null +++ b/tests/CompletionPollConsumerPacketDefaultsTest.php @@ -0,0 +1,39 @@ + ['poll_request_id' => 'first']], ['body' => ['poll_request_id' => 'first']]]; + $same = [['body' => ['poll_request_id' => 'renamed']], ['body' => ['poll_request_id' => 'renamed']]]; + $different = [['body' => ['poll_request_id' => 'first']], ['body' => ['poll_request_id' => 'second']]]; + self::assertSame(CompletionPollPacketDefaults::digest($first), CompletionPollPacketDefaults::digest($same)); + self::assertNotSame(CompletionPollPacketDefaults::digest($first), CompletionPollPacketDefaults::digest($different)); + } + + public function testPayloadChangesCannotHideBehindGeneratedIdsOrDurations(): void + { + $first = [['body' => ['poll_request_id' => 'first', 'result' => ['data' => 'original'], 'duration_ms' => 20]]]; + $second = [['body' => ['poll_request_id' => 'second', 'result' => ['data' => 'changed'], 'duration_ms' => 30]]]; + self::assertNotSame(CompletionPollPacketDefaults::digest($first), CompletionPollPacketDefaults::digest($second)); + } + + public function testTheExistingFixedSocketConsumerKeepsDefaultsIdenticalAtSourceScope(): void + { + $result = CompletionPollPacketDefaults::run(); + self::assertTrue($result['default_option_false']); + self::assertFalse($result['runtime_observed']); + self::assertSame($result['implicit_request_count'], $result['explicit_request_count']); + self::assertMatchesRegularExpression('/\A[a-f0-9]{64}\z/', $result['normalized_packets_sha256']); + } +} diff --git a/tests/CompletionPollConsumerRelayFaultsTest.php b/tests/CompletionPollConsumerRelayFaultsTest.php new file mode 100644 index 00000000..41d6ae7c --- /dev/null +++ b/tests/CompletionPollConsumerRelayFaultsTest.php @@ -0,0 +1,66 @@ +file = tempnam(sys_get_temp_dir(), 'completion-fault-'); + file_put_contents($this->file, json_encode(['events' => [], 'scenario' => 'carrier-loss-activity', + 'prefix' => 'fixture', 'deadline_at' => hrtime(true) / 1e9 + 10], JSON_THROW_ON_ERROR)); + } + + protected function tearDown(): void { unlink($this->file); } + + public function testOnlyOneActualSuccessfulLeasedAnswerIsWithheld(): void + { + $trace = new CompletionPollTrace($this->file); + $answer = ['activity_poll' => ['leased' => true, 'answer' => ['task' => ['task_id' => 'fixture-task']]]]; + $path = '/api/worker/workflow-tasks/parent/complete'; + self::assertFalse(CompletionPollRelayFaults::after($trace, $path, $answer, 500, 'failed')); + self::assertTrue(CompletionPollRelayFaults::after($trace, $path, $answer, 200, 'selected')); + self::assertFalse(CompletionPollRelayFaults::after($trace, $path, $answer, 200, 'retry')); + self::assertSame('selected', $trace->state()['fault_request']); + } + + public function testAnEmptyOrOmittedCarrierCannotSatisfyTheLossControl(): void + { + $trace = new CompletionPollTrace($this->file); + foreach ([[], ['activity_poll' => ['leased' => false]], ['activity_poll' => ['leased' => true]]] as $answer) { + self::assertFalse(CompletionPollRelayFaults::after($trace, '/api/worker/workflow-tasks/parent/complete', $answer, 200, 'not-leased')); + } + self::assertArrayNotHasKey('fault_request', $trace->state()); + } + + public function testWrongCompletionKindDoesNotConsumeASelectedWorkflowLoss(): void + { + $trace = new CompletionPollTrace($this->file); + $trace->change(static function (array &$state): void { $state['scenario'] = 'carrier-loss-workflow'; }); + $answer = ['workflow_poll' => ['leased' => true, 'answer' => ['task' => ['task_id' => 'fixture-task']]]]; + self::assertFalse(CompletionPollRelayFaults::after($trace, '/api/worker/workflow-tasks/parent/complete', $answer, 200, 'wrong-kind')); + self::assertTrue(CompletionPollRelayFaults::after($trace, '/api/worker/activity-tasks/parent/complete', $answer, 200, 'right-kind')); + } + + public function testRawRegistrationTokensAndHistoryDoNotEnterThePublicTrace(): void + { + $token = str_repeat('a', 32); + $answer = CompletionPollTrace::answer(['registration_token' => $token, 'task' => ['task_id' => 'fixture-task', + 'lease_owner' => 'worker', 'lease_expires_at' => '2026-11-07T08:00:30.000000Z', 'history_events' => [['payload' => 'synthetic-payload-canary']]]]); + self::assertSame(hash('sha256', $token), $answer['registration_sha256']); + self::assertArrayNotHasKey('registration_token', $answer); + self::assertStringNotContainsString($token, json_encode($answer, JSON_THROW_ON_ERROR)); + self::assertStringNotContainsString('synthetic-payload-canary', json_encode($answer, JSON_THROW_ON_ERROR)); + self::assertMatchesRegularExpression('/\A[a-f0-9]{64}\z/', $answer['task']['identity_sha256']); + } +} From b24555ccf5ca2485284744df420902965b002351 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Sat, 7 Nov 2026 07:20:27 +0000 Subject: [PATCH 3/3] Add bounded completion-poll load and workflow-refence controls --- phpstan.neon | 5 + .../Support/CompletionPollLoad.php | 246 ++++++++++++++++++ .../Support/CompletionPollLoadEvidence.php | 128 +++++++++ .../Support/CompletionPollLoadProfile.php | 45 ++++ .../Support/CompletionPollOppositeSlots.php | 81 +++--- .../Support/CompletionPollSubject.php | 7 +- .../Support/CompletionPollWorkflowRefence.php | 122 +++++++++ .../compare-completion-poll-load.php | 22 ++ scripts/conformance/completion-polls.md | 91 +++++-- .../conformance/qualify-completion-polls.php | 21 +- tests/CompletionPollConsumerLoadTest.php | 174 +++++++++++++ tests/CompletionPollConsumerSubjectTest.php | 11 + ...pletionPollConsumerWorkflowRefenceTest.php | 62 +++++ 13 files changed, 956 insertions(+), 59 deletions(-) create mode 100644 scripts/conformance/Support/CompletionPollLoad.php create mode 100644 scripts/conformance/Support/CompletionPollLoadEvidence.php create mode 100644 scripts/conformance/Support/CompletionPollLoadProfile.php create mode 100644 scripts/conformance/Support/CompletionPollWorkflowRefence.php create mode 100644 scripts/conformance/compare-completion-poll-load.php create mode 100644 tests/CompletionPollConsumerLoadTest.php create mode 100644 tests/CompletionPollConsumerWorkflowRefenceTest.php diff --git a/phpstan.neon b/phpstan.neon index 0461ed17..e00302f9 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -12,6 +12,11 @@ parameters: - scripts/conformance/Support/CompletionPollMatrixEvidence.php - scripts/conformance/Support/CompletionPollPacketDefaults.php - scripts/conformance/Support/CompletionPollOppositeSlots.php + - scripts/conformance/Support/CompletionPollLoadProfile.php + - scripts/conformance/Support/CompletionPollLoadEvidence.php + - scripts/conformance/Support/CompletionPollLoad.php + - scripts/conformance/Support/CompletionPollWorkflowRefence.php + - scripts/conformance/compare-completion-poll-load.php - scripts/conformance/Support/completion-poll-relay.php excludePaths: analyse: diff --git a/scripts/conformance/Support/CompletionPollLoad.php b/scripts/conformance/Support/CompletionPollLoad.php new file mode 100644 index 00000000..2bf7a8fa --- /dev/null +++ b/scripts/conformance/Support/CompletionPollLoad.php @@ -0,0 +1,246 @@ + */ + public static function run(CompletionPollSubject $subject, string $clientToken, string $workerToken): array + { + $data = $subject->data; + CompletionPollLoadProfile::validate($data); + $workload = CompletionPollLoadProfile::workload($data['scenario']); + $enabled = str_ends_with($data['scenario'], '-on'); + $prefix = 'load-'.bin2hex(random_bytes(8)); + $queue = $prefix.'-queue'; + $directory = sys_get_temp_dir().'/'.$prefix; + $control = new Client($data['endpoint'], namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 2); + $cleanup = new Client($data['endpoint'], namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 1); + if (!mkdir($directory, 0700)) throw new RuntimeException('completion_load_directory'); + $deadline = hrtime(true) / 1e9 + $data['deadline_seconds']; + $children = $handles = $reports = []; + $failure = null; + $finished = 0; + $started = $elapsed = null; + $workerCleanup = false; + $workflowsCleanup = false; + $filesCleanup = false; + $counts = []; + try { + $info = $control->clusterInfo(); + if ($info->version !== $data['server_version']) throw new RuntimeException('completion_runtime_version'); + foreach (CompletionPollEvidence::CAPABILITIES as $capability) { + if (($info->raw['worker_protocol']['server_capabilities'][$capability] ?? null) !== true) { + throw new RuntimeException('completion_load_capability_required'); + } + } + for ($number = 0; $number < CompletionPollLoadProfile::WORKERS; ++$number) { + $pid = pcntl_fork(); + if ($pid === -1) throw new RuntimeException('completion_load_worker_start'); + if ($pid === 0) self::worker($subject, $workerToken, $queue, $prefix.'-'.$number, $directory.'/'.$number.'.json', $enabled); + $children[] = $pid; + } + while (count($control->listWorkers($queue)['workers'] ?? []) !== CompletionPollLoadProfile::WORKERS) { + self::check($deadline, $directory); + usleep(20_000); + } + // Predeclare every owned ID before any start can have an unknown result. + for ($index = 1; $index <= CompletionPollLoadProfile::WORKFLOWS; ++$index) { + $handles[$index] = $control->workflowHandle($prefix.'-workflow-'.$index); + } + $started = hrtime(true); + foreach ($handles as $index => $handle) { + self::check($deadline, $directory); + $handles[$index] = $control->startWorkflow('qualification.load', $handle->workflowId, $queue, + input: [$index], executionTimeoutSeconds: 180, runTimeoutSeconds: 150); + } + $pending = $handles; + while ($pending !== []) { + foreach ($pending as $index => $handle) { + self::check($deadline, $directory); + $status = $handle->describeSelectedRun()->status; + if ($status === 'completed') { + if ($handle->resultOfSelectedRun(timeoutSeconds: 1, pollIntervalSeconds: 0.1) !== $index) { + throw CompletionPollFailure::product('completion_load_workflow_result'); + } + ++$finished; + unset($pending[$index]); + } elseif (in_array($status, ['failed', 'cancelled', 'terminated', 'timed_out'], true)) { + throw CompletionPollFailure::product('completion_load_workflow_result'); + } + } + if ($pending !== []) usleep(20_000); + } + $elapsed = (hrtime(true) - $started) / 1e9; + $workflowsCleanup = true; + } catch (Throwable $error) { + $failure = $error; + if ($started !== null) $elapsed = (hrtime(true) - $started) / 1e9; + } finally { + $cleanupDeadline = hrtime(true) / 1e9 + 30; + try { + self::stop($children, 15); + $workers = $cleanup->listWorkers($queue)['workers'] ?? null; + if (!is_array($workers) || $workers !== []) throw new RuntimeException('completion_worker_registration_survived'); + $workerCleanup = true; + foreach (array_keys($children) as $number) { + $path = $directory.'/'.$number.'.json'; + // No reports are consumed while workers can still write them. + $bytes = @file_get_contents($path, length: 131_073); + if ($bytes === false || strlen($bytes) > 131_072) throw new RuntimeException('completion_load_reports_missing'); + $report = json_decode($bytes, true, flags: JSON_THROW_ON_ERROR); + if (!is_array($report)) throw new RuntimeException('completion_load_report_shape'); + $reports[] = $report; + } + } catch (Throwable $error) { + $failure = new RuntimeException('completion_load_worker_cleanup_failed', previous: $error); + } + if ($failure === null) { + try { + $counts = CompletionPollLoadEvidence::counts($reports); + } catch (Throwable $error) { + $failure = $error; + } + } + if (!$workflowsCleanup) { + $workflowsCleanup = true; + foreach ($handles as $handle) { + if (hrtime(true) / 1e9 >= $cleanupDeadline - 3) { + $workflowsCleanup = false; + break; + } + try { + $description = $cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId); + if (!in_array($description->status, ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { + $cleanup->terminateWorkflow($handle->workflowId, 'Load qualification cleanup', $handle->selectedRunId); + if (!in_array($cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId)->status, + ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) $workflowsCleanup = false; + } + } catch (Throwable $error) { + if ($error instanceof ServerException && $error->status === 404 && $handle->selectedRunId === null) continue; + $workflowsCleanup = false; + } + } + } + for ($number = 0; $number < CompletionPollLoadProfile::WORKERS; ++$number) { + $path = $directory.'/'.$number.'.json'; + if (is_file($path)) unlink($path); + } + $filesCleanup = rmdir($directory); + if (!$workerCleanup || !$workflowsCleanup || !$filesCleanup) $failure = new RuntimeException('completion_load_cleanup_incomplete', previous: $failure); + } + return ['scenario' => $data['scenario'], 'outcome' => $failure === null ? 'pass' : CompletionPollFailure::diagnostic($failure)['outcome'], + 'diagnostic' => $failure === null ? null : CompletionPollFailure::diagnostic($failure), 'load' => $data['load'], + 'workload' => $workload, 'enable_completion_pipelining' => $enabled, + 'measurement' => 'first submission through last verified result; workers registered before measurement', + 'elapsed_seconds' => $elapsed, 'workflow_results_verified' => $finished, 'counts' => $counts, + 'worker_reports' => $reports, 'relay_used' => false, 'request_trace_used' => false, + 'cleanup' => ['complete' => $workerCleanup && $workflowsCleanup && $filesCleanup, + 'worker_groups_and_registrations' => $workerCleanup, 'owned_workflows' => $workflowsCleanup, 'temporary_reports' => $filesCleanup], + 'caller_disposable_runtime_cleanup_required' => true, 'performance_acceptance' => 'not_assessed']; + } + + private static function worker(CompletionPollSubject $subject, string $token, string $queue, string $id, string $reportFile, bool $enabled): never + { + $steps = []; + $callbacks = 0; + $failure = null; + try { + if (posix_setsid() === -1) throw new RuntimeException('completion_worker_process_group'); + $client = new Client($subject->data['endpoint'], namespace: $subject->data['namespace'], workerToken: $token, + requestTimeoutSeconds: 2, workerRequestTimeoutSeconds: 5); + $worker = new Worker($client, $queue, workerId: $id, enableCompletionPipelining: $enabled); + $local = str_contains($subject->data['scenario'], '-local-'); + $worker->registerWorkflow('qualification.load', static function (WorkflowContext $context, mixed ...$arguments) use ($local, &$callbacks): int { + if (count($arguments) !== 1 || !is_int($arguments[0]) || $arguments[0] < 1 || $arguments[0] > CompletionPollLoadProfile::WORKFLOWS) { + throw new RuntimeException('completion_load_arguments'); + } + ++$callbacks; + $index = $arguments[0]; + for ($step = 0; $step < CompletionPollLoadProfile::STEPS; ++$step) { + $answer = $local ? $context->localActivity('qualification.step', [$index, $step]) + : $context->activity('qualification.step', [$index, $step]); + if ($answer !== $index) throw new RuntimeException('completion_load_step_result'); + } + return $index; + }); + $worker->registerActivity('qualification.step', static function (ActivityContext $context, mixed ...$arguments) use (&$steps): int { + if (count($arguments) !== 2 || !is_int($arguments[0]) || !is_int($arguments[1]) + || $arguments[0] < 1 || $arguments[0] > CompletionPollLoadProfile::WORKFLOWS + || $arguments[1] < 0 || $arguments[1] >= CompletionPollLoadProfile::STEPS) throw new RuntimeException('completion_load_arguments'); + $key = $arguments[0].':'.$arguments[1]; + $steps[$key] = ($steps[$key] ?? 0) + 1; + usleep(CompletionPollLoadProfile::STEP_MICROSECONDS); + return $arguments[0]; + }); + $worker->run(1); + } catch (Throwable $error) { + $failure = CompletionPollFailure::diagnostic($error); + } + $bytes = json_encode(['worker_id' => $id, 'outcome' => $failure === null ? 'pass' : 'runner-blocked', + 'diagnostic' => $failure, 'workflow_callbacks' => $callbacks, 'steps' => $steps], JSON_THROW_ON_ERROR); + $written = file_put_contents($reportFile, $bytes) === strlen($bytes); + exit($failure === null && $written ? 0 : 1); + } + + private static function check(float $deadline, string $directory): void + { + if (hrtime(true) / 1e9 >= $deadline) throw new RuntimeException('completion_load_deadline'); + for ($number = 0; $number < CompletionPollLoadProfile::WORKERS; ++$number) { + if (is_file($directory.'/'.$number.'.json')) throw new RuntimeException('completion_load_worker_stopped_early'); + } + } + + /** Signal all owned groups together; the grace is global, not multiplied by worker count. + * @param list $pids + */ + public static function stop(array $pids, float $grace): void + { + $pending = array_fill_keys($pids, true); + foreach ($pids as $pid) { + if (!posix_kill(-$pid, SIGTERM)) posix_kill($pid, SIGTERM); + } + $deadline = hrtime(true) / 1e9 + $grace; + $failed = false; + while ($pending !== []) { + foreach ($pending as $pid => $_) { + $waited = pcntl_waitpid($pid, $status, WNOHANG); + if ($waited !== 0) { + $failed = $failed || $waited === -1 || !pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0; + unset($pending[$pid]); + } + } + if ($pending === []) break; + if (hrtime(true) / 1e9 >= $deadline) { + foreach ($pending as $pid => $_) { + posix_kill(-$pid, SIGKILL); + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status); + } + $failed = true; + break; + } + usleep(10_000); + } + foreach ($pids as $pid) { + if (posix_kill(-$pid, 0)) { + posix_kill(-$pid, SIGKILL); + $failed = true; + } + } + if ($failed) throw new RuntimeException('completion_load_worker_stop_failed'); + } +} diff --git a/scripts/conformance/Support/CompletionPollLoadEvidence.php b/scripts/conformance/Support/CompletionPollLoadEvidence.php new file mode 100644 index 00000000..2f5152ef --- /dev/null +++ b/scripts/conformance/Support/CompletionPollLoadEvidence.php @@ -0,0 +1,128 @@ +> $reports + * @return array + */ + public static function counts(array $reports): array + { + if (count($reports) !== CompletionPollLoadProfile::WORKERS) throw new RuntimeException('completion_load_reports_missing'); + $counts = []; + $callbacks = 0; + foreach ($reports as $report) { + if (($report['outcome'] ?? null) !== 'pass' || !is_array($report['steps'] ?? null) + || !is_int($report['workflow_callbacks'] ?? null) || $report['workflow_callbacks'] < 0) { + throw new RuntimeException('completion_load_worker_failed'); + } + $callbacks += $report['workflow_callbacks']; + foreach ($report['steps'] as $key => $count) { + if (!is_string($key) || preg_match('/\A([1-9][0-9]*):([0-9]+)\z/', $key, $parts) !== 1 + || (int) $parts[1] > CompletionPollLoadProfile::WORKFLOWS || (int) $parts[2] >= CompletionPollLoadProfile::STEPS + || !is_int($count) || $count < 1) throw new RuntimeException('completion_load_report_shape'); + $counts[$key] = ($counts[$key] ?? 0) + $count; + } + } + $expected = CompletionPollLoadProfile::WORKFLOWS * CompletionPollLoadProfile::STEPS; + if (count($counts) !== $expected || array_sum($counts) !== $expected || $callbacks < CompletionPollLoadProfile::WORKFLOWS) { + throw CompletionPollFailure::product('completion_load_functional_counts'); + } + return ['unique_steps' => count($counts), 'step_callbacks' => array_sum($counts), 'workflow_callbacks' => $callbacks]; + } + + /** Authenticate and parse the same bounded result bytes. No package/autoload/network operation. + * @return array + */ + public static function read(string $path, string $sha256): array + { + if (preg_match('/\A[a-f0-9]{64}\z/', $sha256) !== 1 || !stream_is_local($path)) throw new RuntimeException('completion_load_result_identity'); + $file = @fopen($path, 'rb'); + if ($file === false) throw new RuntimeException('completion_load_result_identity'); + try { + $stat = fstat($file); + $bytes = stream_get_contents($file, 1_048_577); + if ($stat === false || ($stat['mode'] & 0170000) !== 0100000 || $bytes === false || strlen($bytes) > 1_048_576 + || !hash_equals($sha256, hash('sha256', $bytes))) throw new RuntimeException('completion_load_result_identity'); + $data = json_decode($bytes, true, flags: JSON_THROW_ON_ERROR); + if (!is_array($data) || array_is_list($data)) throw new RuntimeException('completion_load_result_shape'); + return $data; + } finally { + fclose($file); + } + } + + /** Three OFF and three ON results for exactly one tuple/backend/workload/conditions binding. + * @param list> $results + * @return array + */ + public static function compare(array $results): array + { + if (count($results) !== 6) throw new RuntimeException('completion_load_six_results_required'); + $binding = null; + $databases = $subjects = $values = []; + foreach ($results as $result) { + $cell = $result['result'] ?? []; + $scenario = $cell['scenario'] ?? ''; + CompletionPollLoadProfile::validate(['scenario' => $scenario, 'load' => $cell['load'] ?? null]); + if (!in_array($scenario, CompletionPollLoadProfile::SCENARIOS, true) + || ($result['schema'] ?? null) !== 'durable-workflow.php-completion-polls-result/v1' + || ($result['outcome'] ?? null) !== 'pass' || ($cell['outcome'] ?? null) !== 'pass' + || ($cell['workload'] ?? null) !== CompletionPollLoadProfile::workload($scenario) + || ($cell['workflow_results_verified'] ?? null) !== CompletionPollLoadProfile::WORKFLOWS + || ($cell['enable_completion_pipelining'] ?? null) !== str_ends_with($scenario, '-on') + || ($cell['relay_used'] ?? null) !== false || ($cell['request_trace_used'] ?? null) !== false + || ($cell['counts']['unique_steps'] ?? null) !== CompletionPollLoadProfile::WORKFLOWS * CompletionPollLoadProfile::STEPS + || ($cell['counts']['step_callbacks'] ?? null) !== CompletionPollLoadProfile::WORKFLOWS * CompletionPollLoadProfile::STEPS + || ($cell['cleanup']['complete'] ?? null) !== true + || (!is_float($cell['elapsed_seconds'] ?? null) && !is_int($cell['elapsed_seconds'] ?? null)) + || !is_finite((float) $cell['elapsed_seconds']) || $cell['elapsed_seconds'] <= 0) { + throw new RuntimeException('completion_load_result_not_qualified'); + } + $identity = $result['identity'] ?? []; + foreach (['sdk_reference', 'caller_runner_reference'] as $key) { + if (!is_string($identity[$key] ?? null) || preg_match('/\A[a-f0-9]{40}\z/', $identity[$key]) !== 1) throw new RuntimeException('completion_load_result_identity'); + } + foreach (['lock_sha256', 'subject_sha256'] as $key) { + if (!is_string($identity[$key] ?? null) || preg_match('/\A[a-f0-9]{64}\z/', $identity[$key]) !== 1) throw new RuntimeException('completion_load_result_identity'); + } + if (!is_string($identity['sdk_version'] ?? null) || !is_string($result['server_version'] ?? null) + || !is_string($result['caller_server_image'] ?? null) + || preg_match('~\A(?:ghcr\.io/durable-workflow/server|durableworkflow/server)@sha256:[a-f0-9]{64}\z~', $result['caller_server_image']) !== 1) { + throw new RuntimeException('completion_load_result_identity'); + } + $selected = ['sdk_version' => $identity['sdk_version'], 'sdk_reference' => $identity['sdk_reference'], + 'runner_reference' => $identity['caller_runner_reference'], 'lock_sha256' => $identity['lock_sha256'], + 'server_version' => $result['server_version'], 'server_image' => $result['caller_server_image'], + 'workload' => $cell['workload'], 'backend' => $cell['load']['backend'], 'conditions_sha256' => $cell['load']['conditions_sha256']]; + if ($binding !== null && $selected !== $binding) throw new RuntimeException('completion_load_mixed_bindings'); + $binding = $selected; + $database = $cell['load']['database_identity']; + if (isset($databases[$database]) || isset($subjects[$identity['subject_sha256']])) throw new RuntimeException('completion_load_reused_cell'); + $databases[$database] = $subjects[$identity['subject_sha256']] = true; + $mode = str_ends_with($scenario, '-on') ? 'on' : 'off'; + $repetition = $cell['load']['repetition']; + if (isset($values[$mode][$repetition])) throw new RuntimeException('completion_load_duplicate_repetition'); + $values[$mode][$repetition] = $cell['elapsed_seconds']; + } + foreach (['off', 'on'] as $mode) { + if (count($values[$mode] ?? []) !== 3) throw new RuntimeException('completion_load_repetitions_missing'); + ksort($values[$mode]); + } + $off = array_sum($values['off']) / 3; + $on = array_sum($values['on']) / 3; + return ['scope' => 'descriptive comparison; caller publication, fresh database and host/resource evidence remain separate', + 'binding' => $binding, 'elapsed_seconds' => array_map('array_values', $values), + 'mean_seconds' => ['off' => $off, 'on' => $on], 'off_over_on_ratio' => $off / $on, + 'on_slower_than_off' => $on > $off, 'performance_acceptance' => 'not_assessed']; + } +} diff --git a/scripts/conformance/Support/CompletionPollLoadProfile.php b/scripts/conformance/Support/CompletionPollLoadProfile.php new file mode 100644 index 00000000..9c8f0cac --- /dev/null +++ b/scripts/conformance/Support/CompletionPollLoadProfile.php @@ -0,0 +1,45 @@ + $data */ + public static function validate(array $data): void + { + $load = $data['load'] ?? null; + if (!in_array($data['scenario'] ?? null, self::SCENARIOS, true)) { + if (array_key_exists('load', $data)) throw new RuntimeException('completion_subject_load'); + return; + } + $keys = ['backend', 'database_identity', 'repetition', 'conditions_sha256']; + if (!is_array($load) || array_diff(array_keys($load), $keys) !== [] || array_diff($keys, array_keys($load)) !== [] + || !in_array($load['backend'] ?? null, ['sqlite', 'mysql', 'postgresql'], true) + || !is_string($load['database_identity'] ?? null) + || preg_match('/\A[A-Za-z0-9][A-Za-z0-9_-]{7,63}\z/', $load['database_identity']) !== 1 + || !is_int($load['repetition'] ?? null) || $load['repetition'] < 1 || $load['repetition'] > 3 + || !is_string($load['conditions_sha256'] ?? null) || preg_match('/\A[a-f0-9]{64}\z/', $load['conditions_sha256']) !== 1) { + throw new RuntimeException('completion_subject_load'); + } + } + + /** @return array */ + public static function workload(string $scenario): array + { + if (!in_array($scenario, self::SCENARIOS, true)) throw new RuntimeException('completion_subject_load'); + return ['profile' => 'six-workers-200-workflows-five-20ms-steps/v1', 'workers' => self::WORKERS, + 'workflows' => self::WORKFLOWS, 'steps_per_workflow' => self::STEPS, 'step_microseconds' => self::STEP_MICROSECONDS, + 'activity_kind' => str_contains($scenario, '-local-') ? 'local' : 'remote']; + } +} diff --git a/scripts/conformance/Support/CompletionPollOppositeSlots.php b/scripts/conformance/Support/CompletionPollOppositeSlots.php index 31b2cf50..72475add 100644 --- a/scripts/conformance/Support/CompletionPollOppositeSlots.php +++ b/scripts/conformance/Support/CompletionPollOppositeSlots.php @@ -13,6 +13,7 @@ require_once __DIR__.'/CompletionPollSubject.php'; require_once __DIR__.'/CompletionPollTrace.php'; require_once __DIR__.'/CompletionPollFailure.php'; +require_once __DIR__.'/CompletionPollWorkflowRefence.php'; require_once dirname(__DIR__, 3).'/tests/Support/CompletionPipelineScenario.php'; /** Public installed Client calls only. No administrative clock, SQL or producer implementation. */ @@ -23,11 +24,12 @@ public static function run(CompletionPollSubject $subject, string $clientToken, { return CompletionPipelineScenario::run(static function (string $relay, string $file) use ($subject, $clientToken, $workerToken): array { $data = $subject->data; - $prefix = 'opposite-'.bin2hex(random_bytes(8)); + $refence = $data['scenario'] === 'workflow-refence'; + $prefix = ($refence ? 'refence-' : 'opposite-').bin2hex(random_bytes(8)); $worker = $prefix.'-worker'; $queue = $prefix.'-queue'; file_put_contents($file, json_encode(['events' => [], 'endpoint' => $data['endpoint'], - 'scenario' => 'opposite-slots', 'prefix' => $prefix], JSON_THROW_ON_ERROR)); + 'scenario' => $data['scenario'], 'prefix' => $prefix], JSON_THROW_ON_ERROR)); $trace = new CompletionPollTrace($file); $control = new Client($relay, namespace: $data['namespace'], controlToken: $clientToken, requestTimeoutSeconds: 2); $actor = new Client($relay, namespace: $data['namespace'], workerToken: $workerToken, workerRequestTimeoutSeconds: 2); @@ -41,7 +43,7 @@ public static function run(CompletionPollSubject $subject, string $clientToken, $assertions = []; $terminated = 0; $handles = []; - for ($i = 1; $i <= 3; ++$i) $handles[$i] = $control->workflowHandle($prefix.'-'.$i); + for ($i = 1; $i <= ($refence ? 1 : 3); ++$i) $handles[$i] = $control->workflowHandle($prefix.'-'.$i); try { if ($control->clusterInfo()->version !== $data['server_version']) throw new RuntimeException('completion_runtime_version'); $registered = $actor->registerWorker($worker, $queue, ['qualification.wire'], ['qualification.remote'], @@ -58,34 +60,41 @@ public static function run(CompletionPollSubject $subject, string $clientToken, $handles[$index] = $control->startWorkflow('qualification.wire', $handle->workflowId, $queue, input: [$index], executionTimeoutSeconds: 180, runTimeoutSeconds: 150); } - $workflowOne = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf1'), $worker, 'workflow'); - $workflowTwo = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf2'), $worker, 'workflow'); - if ($workflowOne['task_id'] === $workflowTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); - $check(); - $activityId = $prefix.'-carried-activity'; - $answer = $actor->completeWorkflowTask($workflowOne['task_id'], $worker, $workflowOne['workflow_task_attempt'], - self::schedule($actor), activityPoll: self::carrier($worker, $queue, $activityId)); - self::omitted($answer, 'activity_poll'); - // The other original workflow is still leased. A same-ID - // ordinary activity poll proves eligible work was present. - $activityOne = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $activityId), $worker, 'activity'); - $actor->completeWorkflowTask($workflowTwo['task_id'], $worker, $workflowTwo['workflow_task_attempt'], self::schedule($actor)); - $activityTwo = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $prefix.'-act2'), $worker, 'activity'); - if ($activityOne['task_id'] === $activityTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); - $check(); - $workflowId = $prefix.'-carried-workflow'; - $workflowCarrier = self::carrier($worker, $queue, $workflowId) + ['history_page_size' => 100, 'task_kinds' => ['workflow']]; - $answer = $actor->completeActivityTask($activityTwo['task_id'], $activityTwo['activity_attempt_id'], $worker, 2, - workflowPoll: $workflowCarrier); - self::omitted($answer, 'workflow_poll'); - // There is both a third original workflow and the activity's - // next workflow task ready. Release the held activity normally, - // then recover with the omitted carrier's exact original ID. - $actor->completeActivityTask($activityOne['task_id'], $activityOne['activity_attempt_id'], $worker, 1); - $recovered = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $workflowId), $worker, 'workflow'); - $assertions = ['scope' => 'installed Client public-API opposite-slot control; managed Worker capacity is a separate row', - 'live_workflow_declined_activity_carrier' => true, 'live_activity_declined_workflow_carrier' => true, - 'eligible_work_and_original_id_followup' => true, 'recovered_workflow_task' => CompletionPollTrace::task($recovered)]; + if ($refence) { + $assertions = CompletionPollWorkflowRefence::run($actor, $worker, $queue, $prefix, $check, $token); + if ($handles[1]->resultOfSelectedRun(timeoutSeconds: 1, pollIntervalSeconds: 0.1) !== 1) { + throw CompletionPollFailure::product('completion_refence_workflow_result'); + } + } else { + $workflowOne = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf1'), $worker, 'workflow'); + $workflowTwo = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-wf2'), $worker, 'workflow'); + if ($workflowOne['task_id'] === $workflowTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); + $check(); + $activityId = $prefix.'-carried-activity'; + $answer = $actor->completeWorkflowTask($workflowOne['task_id'], $worker, $workflowOne['workflow_task_attempt'], + self::schedule($actor), activityPoll: self::carrier($worker, $queue, $activityId)); + self::omitted($answer, 'activity_poll'); + // The other original workflow is still leased. A same-ID + // ordinary activity poll proves eligible work was present. + $activityOne = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $activityId), $worker, 'activity'); + $actor->completeWorkflowTask($workflowTwo['task_id'], $worker, $workflowTwo['workflow_task_attempt'], self::schedule($actor)); + $activityTwo = self::claim($actor->pollActivityTaskResponse($worker, $queue, 0, $prefix.'-act2'), $worker, 'activity'); + if ($activityOne['task_id'] === $activityTwo['task_id']) throw CompletionPollFailure::product('completion_opposite_distinct_leases'); + $check(); + $workflowId = $prefix.'-carried-workflow'; + $workflowCarrier = self::carrier($worker, $queue, $workflowId) + ['history_page_size' => 100, 'task_kinds' => ['workflow']]; + $answer = $actor->completeActivityTask($activityTwo['task_id'], $activityTwo['activity_attempt_id'], $worker, 2, + workflowPoll: $workflowCarrier); + self::omitted($answer, 'workflow_poll'); + // There is both a third original workflow and the activity's + // next workflow task ready. Release the held activity normally, + // then recover with the omitted carrier's exact original ID. + $actor->completeActivityTask($activityOne['task_id'], $activityOne['activity_attempt_id'], $worker, 1); + $recovered = self::claim($actor->pollWorkflowTaskResponse($worker, $queue, 0, $workflowId), $worker, 'workflow'); + $assertions = ['scope' => 'installed Client public-API opposite-slot control; managed Worker capacity is a separate row', + 'live_workflow_declined_activity_carrier' => true, 'live_activity_declined_workflow_carrier' => true, + 'eligible_work_and_original_id_followup' => true, 'recovered_workflow_task' => CompletionPollTrace::task($recovered)]; + } } catch (Throwable $error) { $failure = $error; } finally { @@ -116,10 +125,14 @@ public static function run(CompletionPollSubject $subject, string $clientToken, } } } - if ($failure === null && ($cleanup->listWorkers($queue)['workers'] ?? []) !== []) { - $failure = new RuntimeException('completion_worker_registration_survived'); + if ($failure === null) { + $workers = $cleanup->listWorkers($queue)['workers'] ?? null; + if (!is_array($workers) || $workers !== []) $failure = new RuntimeException('completion_worker_registration_survived'); + if (array_filter($trace->state()['events'], static fn (array $event): bool => $event['kind'] === 'relay_error') !== []) { + $failure = new RuntimeException('completion_relay_failed'); + } } - return ['scenario' => 'opposite-slots', 'outcome' => $failure === null ? 'pass' : CompletionPollFailure::diagnostic($failure)['outcome'], + return ['scenario' => $data['scenario'], 'outcome' => $failure === null ? 'pass' : CompletionPollFailure::diagnostic($failure)['outcome'], 'diagnostic' => $failure === null ? null : CompletionPollFailure::diagnostic($failure), 'assertions' => $assertions, 'workflows_intentionally_terminated' => $terminated, 'events' => $trace->state()['events']]; }, __DIR__.'/completion-poll-relay.php', true); diff --git a/scripts/conformance/Support/CompletionPollSubject.php b/scripts/conformance/Support/CompletionPollSubject.php index c46bf51f..2a8601ff 100644 --- a/scripts/conformance/Support/CompletionPollSubject.php +++ b/scripts/conformance/Support/CompletionPollSubject.php @@ -8,13 +8,16 @@ use ReflectionClass; use RuntimeException; +require_once __DIR__.'/CompletionPollLoadProfile.php'; + /** Caller-selected identity, not a package installer or runtime admission authority. */ final class CompletionPollSubject { public const SCHEMA = 'durable-workflow.php-completion-polls-subject/v1'; public const SCENARIOS = ['carriers', 'pipeline', 'pipeline-loss', 'fallback', 'capacity', 'carrier-loss-activity', 'carrier-loss-workflow', 'carrier-loss-workflow-completion', 'completion-transit', - 'carrier-expiry', 'carrier-refence', 'pipeline-buffer-age', 'prefetch-loss', 'default-off', 'explicit-off', 'packet-defaults', 'opposite-slots']; + 'carrier-expiry', 'carrier-refence', 'pipeline-buffer-age', 'prefetch-loss', 'default-off', 'explicit-off', 'packet-defaults', 'opposite-slots', 'workflow-refence', + ...CompletionPollLoadProfile::SCENARIOS]; /** @param array $data */ private function __construct(public readonly array $data, public readonly string $vendor, public readonly string $sha256) @@ -49,6 +52,7 @@ public static function read(string $autoload, string $packet, string $sha256): s $data = json_decode($bytes, true, flags: JSON_THROW_ON_ERROR); $fields = ['schema', 'runner_reference', 'sdk_version', 'sdk_reference', 'lock_sha256', 'server_version', 'server_image', 'endpoint', 'namespace', 'scenario', 'deadline_seconds']; + if (is_array($data) && in_array($data['scenario'] ?? null, CompletionPollLoadProfile::SCENARIOS, true)) $fields[] = 'load'; if (!is_array($data) || array_is_list($data) || ($data['schema'] ?? null) !== self::SCHEMA || array_diff(array_keys($data), $fields) !== [] || array_diff($fields, array_keys($data)) !== []) { throw new RuntimeException('completion_subject_shape'); @@ -71,6 +75,7 @@ public static function read(string $autoload, string $packet, string $sha256): s throw new RuntimeException('completion_subject_runtime'); } self::loopback($data['endpoint']); + CompletionPollLoadProfile::validate($data); $vendor = realpath(dirname($autoload)); if ($vendor === false || basename($vendor) !== 'vendor' || realpath($autoload) !== $vendor.'/autoload.php') { throw new RuntimeException('completion_subject_autoload'); diff --git a/scripts/conformance/Support/CompletionPollWorkflowRefence.php b/scripts/conformance/Support/CompletionPollWorkflowRefence.php new file mode 100644 index 00000000..2f0a6dcf --- /dev/null +++ b/scripts/conformance/Support/CompletionPollWorkflowRefence.php @@ -0,0 +1,122 @@ + + */ + public static function run(Client $actor, string $worker, string $queue, string $prefix, callable $check, string &$token): array + { + $first = $actor->pollWorkflowTaskResponse($worker, $queue, 0, $prefix.'-parent')['task'] ?? null; + if (!is_array($first)) throw new RuntimeException('completion_refence_initial_claim'); + $scheduled = $actor->completeWorkflowTask($first['task_id'], $worker, $first['workflow_task_attempt'], + [['type' => 'schedule_activity', 'activity_type' => 'qualification.remote', + 'arguments' => $actor->payloadCodec()->envelope([1]), 'start_to_close_timeout' => 120]]); + if (($scheduled['recorded'] ?? null) !== true) throw CompletionPollFailure::product('completion_refence_initial_completion'); + $activity = $actor->pollActivityTaskResponse($worker, $queue, 0, $prefix.'-activity')['task'] ?? null; + if (!is_array($activity)) throw new RuntimeException('completion_refence_initial_claim'); + $pollId = $prefix.'-original-workflow'; + $carriedStarted = hrtime(true); + $answer = $actor->completeActivityTask($activity['task_id'], $activity['activity_attempt_id'], $worker, 1, + workflowPoll: ['worker_id' => $worker, 'task_queue' => $queue, 'poll_request_id' => $pollId, + 'timeout_seconds' => 0, 'history_page_size' => 100, 'task_kinds' => ['workflow']]); + $original = $answer['workflow_poll']['answer']['task'] ?? null; + if (($answer['recorded'] ?? null) !== true || ($answer['workflow_poll']['leased'] ?? null) !== true + || !is_array($original) || ($original['lease_owner'] ?? null) !== $worker + || !is_int($original['workflow_task_attempt'] ?? null) || !is_array($original['history_events'] ?? null)) { + throw CompletionPollFailure::product('completion_refence_carried_workflow_missing'); + } + $check(); + $oldDigest = hash('sha256', $token); + $replacement = $actor->registerWorker($worker, $queue, ['qualification.wire'], ['qualification.remote'], + maxConcurrentWorkflowTasks: 2, maxConcurrentActivityTasks: 2); + $replacementToken = $replacement['registration_token'] ?? null; + if (($replacement['registered'] ?? null) !== true || ($replacement['worker_id'] ?? null) !== $worker + || ($replacement['task_queue'] ?? null) !== $queue || !is_string($replacementToken) + || preg_match('/\A[a-f0-9]{32}\z/', $replacementToken) !== 1) throw new RuntimeException('completion_refence_receipt_required'); + // Transfer cleanup authority immediately after a verified response. + $token = $replacementToken; + if (hash_equals($oldDigest, hash('sha256', $token))) throw CompletionPollFailure::product('completion_refence_token_not_rotated'); + // Charge the whole original completion and registration transit. A + // slow cell cannot use expiry as evidence of registration revocation. + if ((hrtime(true) - $carriedStarted) / 1e9 >= CompletionPollFaults::window($answer['workflow_poll']['answer']) - 5) { + throw new RuntimeException('completion_refence_live_window_required'); + } + $commands = [['type' => 'complete_workflow', 'result' => $actor->payloadCodec()->envelope(1)]]; + $heartbeat = static fn (): array => $actor->heartbeatWorkflowTask($original['task_id'], $worker, $original['workflow_task_attempt']); + $completion = static fn (): array => $actor->completeWorkflowTask($original['task_id'], $worker, $original['workflow_task_attempt'], $commands); + foreach ([$heartbeat, $completion] as $index => $operation) { + $check(); + self::refused($operation, $original, 'task_not_leased', null, $index === 1); + } + $check(); + $recovered = $actor->pollWorkflowTaskResponse($worker, $queue, 0, $pollId, historyPageSize: 100)['task'] ?? null; + if (!is_array($recovered)) throw CompletionPollFailure::product('completion_refence_recovery_missing'); + self::recovered($original, $recovered); + foreach ([$heartbeat, $completion] as $index => $operation) { + $check(); + self::refused($operation, $original, 'workflow_task_attempt_mismatch', $recovered['workflow_task_attempt'], $index === 1); + } + $check(); + $confirmed = $actor->heartbeatWorkflowTask($recovered['task_id'], $worker, $recovered['workflow_task_attempt']); + if (($confirmed['renewed'] ?? null) !== true) throw CompletionPollFailure::product('completion_refence_new_authority_lost'); + $completed = $actor->completeWorkflowTask($recovered['task_id'], $worker, $recovered['workflow_task_attempt'], $commands); + if (($completed['recorded'] ?? null) !== true) throw CompletionPollFailure::product('completion_refence_new_authority_lost'); + return ['registration_rotated' => true, 'pre_recovery_reason' => 'task_not_leased', + 'post_recovery_reason' => 'workflow_task_attempt_mismatch', 'original_id_fresh_claim' => true, + 'original_attempt' => $original['workflow_task_attempt'], 'current_attempt' => $recovered['workflow_task_attempt'], + 'history_preserved' => true, 'current_authority_completed' => true, + 'old_registration_sha256' => $oldDigest, 'registration_sha256' => hash('sha256', $token)]; + } + + /** @param array $original + * @param array $current + */ + public static function recovered(array $original, array $current): void + { + foreach (['task_id', 'workflow_id', 'run_id', 'lease_owner', 'workflow_type', 'input', 'history_events'] as $key) { + if (($original[$key] ?? null) !== ($current[$key] ?? null)) throw CompletionPollFailure::product('completion_refence_recovery_changed'); + } + if (!is_array($original['history_events'] ?? null) || !is_int($original['workflow_task_attempt'] ?? null) + || ($current['workflow_task_attempt'] ?? null) !== $original['workflow_task_attempt'] + 1 + || !is_string($current['lease_expires_at'] ?? null)) throw CompletionPollFailure::product('completion_refence_recovery_changed'); + } + + /** @param callable(): array $operation + * @param array $original + */ + private static function refused(callable $operation, array $original, string $reason, ?int $attempt, bool $completion): void + { + try { + $operation(); + } catch (ServerException $error) { + self::refusal($error, $original, $reason, $attempt, $completion); + return; + } + throw CompletionPollFailure::product('completion_refence_old_authority_accepted'); + } + + /** @param array $original */ + public static function refusal(ServerException $error, array $original, string $reason, ?int $attempt, bool $completion): void + { + $details = $error->details ?? []; + if ($error->status !== 409 || $error->reason !== $reason || ($details['task_id'] ?? null) !== $original['task_id'] + || ($details['workflow_task_attempt'] ?? null) !== $original['workflow_task_attempt'] + || ($attempt === null ? ($details['task_status'] ?? null) !== 'ready' : ($details['current_attempt'] ?? null) !== $attempt) + || ($completion && $attempt === null && ($details['recorded'] ?? null) !== false)) { + throw CompletionPollFailure::product('completion_refence_wrong_refusal'); + } + } +} diff --git a/scripts/conformance/compare-completion-poll-load.php b/scripts/conformance/compare-completion-poll-load.php new file mode 100644 index 00000000..c00ab5b8 --- /dev/null +++ b/scripts/conformance/compare-completion-poll-load.php @@ -0,0 +1,22 @@ + 'durable-workflow.php-completion-polls-load-comparison/v1', + 'result_sha256' => $digests, 'comparison' => CompletionPollLoadEvidence::compare($results)], JSON_THROW_ON_ERROR).PHP_EOL; +} catch (Throwable $error) { + fwrite(STDERR, json_encode(CompletionPollFailure::diagnostic($error), JSON_THROW_ON_ERROR).PHP_EOL); + exit(1); +} diff --git a/scripts/conformance/completion-polls.md b/scripts/conformance/completion-polls.md index 3be7853b..8e0b0570 100644 --- a/scripts/conformance/completion-polls.md +++ b/scripts/conformance/completion-polls.md @@ -13,8 +13,10 @@ The template has unselected identities and refuses before consumer autoload. The program does not install packages, fetch tools, start Server, upgrade a database or change accounts/credentials. It requires an already-running caller-owned disposable runtime at an explicit `http://127.0.0.1:port` address. -It creates only uniquely named synthetic workflows and one SDK Worker under -the supplied namespace. The caller must own that runtime and namespace and +It creates uniquely named synthetic workflows and one SDK Worker, or six +Workers in an explicitly selected load cell, under the supplied namespace. +The public-API opposite-slot and workflow-refence rows use an owned registration +instead of a managed Worker. The caller must own that runtime and namespace and include this effect in its separately selected execution packet. A loopback address alone does not establish operating permission. @@ -69,8 +71,8 @@ The basic scenarios are: producer lacks support. The controlled answer delay and loss are identified explicitly. They are not -an undelayed load comparison or a speedup measurement. The additional controls below are connected at Source scope. Undelayed load -support is still being prepared. Every selected row needs its own actual +an undelayed load comparison or a speedup measurement. The additional controls below, including undelayed load support, are connected +at Source scope. Every selected row needs its own actual published-artifact result before the #328–#332 matrix is complete. Each invocation has a selected 15–120 second product deadline, bounded HTTP @@ -103,9 +105,9 @@ fixed HTTP data to test byte/role forwarding, error preservation and one successful-answer loss. They create no Server, package installation, profile or published-product acceptance. -The remaining matrix is being prepared against the same accepted schema63 -API. All controls except the pending load cells have executable selections. -They remain unexecuted against an installed package and Native runtime. Delay selection uses the +The remaining matrix uses the same accepted schema63 API. All controls below +have executable selections and remain unexecuted against an installed package +and Native runtime. Delay selection uses the actual normal-poll `server_time` and original `lease_expires_at`. An absent clock, unsuitable grant window or insufficient caller budget blocks that control rather than substituting a worker clock or changing Server state. @@ -116,19 +118,70 @@ control rather than substituting a worker clock or changing Server state. | `completion-transit` | Delay the real completion request before its carried claim. Require original-request charging and confirmation before callback when the reported reserve is insufficient. | | `carrier-expiry` | Hold the real answer past its reported original expiry. Require fenced confirmation/refusal and no callback or publication under that expired generation. | | `carrier-refence` | Re-register this invocation's same synthetic worker through the public worker API. Retain only token digests and require original activity-poll recovery to return the actual `poll_cached_task_expired` refusal; clean the owned replacement with its actual fenced receipt. | +| `workflow-refence` | Obtain a carried workflow task, replace the owned registration, check old operations before recovery, recover the original poll ID as a new attempt, and check old operations after recovery. Complete with the new authority. | | `pipeline-buffer-age` | Delay a real registration-heartbeat reply after prefetch. Require workflow lease confirmation before the aged task executes. Registration heartbeat does not renew task authority. | | `opposite-slots`, `capacity` | Hold actual opposite-kind leases through public polls/completions. Observe real carrier omission, then reservation release and later work without disabling the capability. | | `prefetch-loss` | Withhold one real successful prefetch answer. Preserve the original ID, backoff and lease/history on recovery without another reservation. | | `default-off`, `explicit-off`, `packet-defaults` | Exercise omitted and explicit-false options against the actual runtime. Keep the installed fixed-socket packet regression separate from native producer acceptance. | -| Bounded load | Direct runtime calls with no relay delay or per-request trace. Compare explicit OFF/ON cells for six workers, 200 workflows and five 20ms local or remote steps. Retain all three repetitions and regressions. | - -The registration control does not assert that replacement revokes an already -granted workflow lease. That distinct producer vector is requested in Server -#1067. `packet-defaults` runs the installed SDK against the fixed HTTP fixture -only and explicitly reports `runtime_observed: false`. `opposite-slots` uses -public Client polls/completions rather than managed Worker capacity. - -Load cells require caller evidence for fresh isolated databases and the actual -backend, image, host and resource settings. The runner does not create a -database, invent an administrative reset or infer backend identity from a -label. Descriptive timings alone are not a speedup or release acceptance. +| `load-local-off`, `load-local-on`, `load-remote-off`, `load-remote-on` | Direct runtime calls with no relay delay or per-request trace. Compare explicit OFF/ON cells for six workers, 200 workflows and five 20ms local or remote steps. Retain all three repetitions and regressions. | + +The activity registration row and workflow registration row have distinct +contracts. The accepted [Server #1067 vector](https://github.com/durable-workflow/server/issues/1067#issuecomment-6703522814) +binds registration's atomic workflow-lease release. `workflow-refence` requires +`task_not_leased` before recovery and `workflow_task_attempt_mismatch` after +the same original poll ID freshly claims attempt n+1. History must remain +unchanged before that new attempt completes. No per-grant incarnation field +is invented. This is producer Source authority, not an installed result. + +`packet-defaults` runs the installed SDK against the fixed HTTP fixture only +and explicitly reports `runtime_observed: false`. `opposite-slots` uses public +Client polls/completions rather than managed Worker capacity. + +Load cells make direct runtime requests. They start no relay and record no +per-request trace. All six Workers register before measurement. Timing spans +the first sequential submission through the last verified result, including +control requests and polling. It is not a Server-internal last-close timing. +Each cell checks all 200 results and all 1,000 logical activity executions. +Worker-local counters are written once after shutdown. Remote workflow replay +may invoke the workflow handler repeatedly, while each logical step must +execute exactly once in this healthy, fault-free workload. + +For a load scenario only, the sealed subject must additionally contain this +closed object, with the actual caller bindings in place of these examples: + +```json +"load": { + "backend": "sqlite", + "database_identity": "fresh-isolated-cell-identity", + "repetition": 1, + "conditions_sha256": "" +} +``` + +Backend is `sqlite`, `mysql` or `postgresql`; repetition is 1, 2 or 3. Use a +fresh isolated database and a unique subject/database identity for each cell. +The conditions evidence must bind actual host, architecture, CPU/RAM limits, +backend version and durability settings, plus the unchanged runtime options. +The runner does not create/reset databases or infer backend identity from a +label. Caller runtime/database teardown remains mandatory, including after a +cleanup failure. All Worker groups receive shutdown together under one global +15 second grace within the 30 second cleanup tail. An incomplete termination +sweep on a failed 200-workflow cell is explicitly blocked, never accepted. + +Compare six individually hashed result files, three OFF and three ON, for one +backend and workload: + +```sh +php scripts/conformance/compare-completion-poll-load.php \ + /results/off-1.json "$OFF_1_SHA256" /results/off-2.json "$OFF_2_SHA256" \ + /results/off-3.json "$OFF_3_SHA256" /results/on-1.json "$ON_1_SHA256" \ + /results/on-2.json "$ON_2_SHA256" /results/on-3.json "$ON_3_SHA256" +``` + +The comparator reads each bounded file once and binds the parsed bytes to its +hash. It rejects mixed tuple/workload/backend/conditions bindings, reused cell +identities, incomplete cleanup and missing/duplicate repetitions. It retains +all six times, both means, their ratio and whether ON was slower. It imposes +no performance pass threshold and reports `performance_acceptance: not_assessed`. +Publication, fresh-database and host/resource evidence remain caller-owned. +Descriptive timing alone is not a speedup or release acceptance. diff --git a/scripts/conformance/qualify-completion-polls.php b/scripts/conformance/qualify-completion-polls.php index b36dc6c3..e5f3fddf 100644 --- a/scripts/conformance/qualify-completion-polls.php +++ b/scripts/conformance/qualify-completion-polls.php @@ -8,6 +8,8 @@ use DurableWorkflow\Qualification\CompletionPollFailure; use DurableWorkflow\Qualification\CompletionPollPacketDefaults; use DurableWorkflow\Qualification\CompletionPollOppositeSlots; +use DurableWorkflow\Qualification\CompletionPollLoad; +use DurableWorkflow\Qualification\CompletionPollLoadProfile; use DurableWorkflow\Qualification\CompletionPollSubject; use DurableWorkflow\Qualification\CompletionPollTrace; use DurableWorkflow\Tests\Support\CompletionPipelineScenario; @@ -245,8 +247,9 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, } if ($waited === -1) { $failure = new RuntimeException('completion_worker_reap_failed', previous: $failure); - } elseif (!pcntl_wifexited($status) || pcntl_wexitstatus($status) !== 0) { - if (!$expectedStop) $failure ??= new RuntimeException('completion_worker_failed'); + } elseif (!pcntl_wifexited($status) || (pcntl_wexitstatus($status) !== 0 + && !($expectedStop && pcntl_wexitstatus($status) === 1))) { + $failure ??= new RuntimeException('completion_worker_failed'); } if (posix_kill(-$pid, 0)) { posix_kill(-$pid, SIGKILL); @@ -254,7 +257,7 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, } if ($failure !== null || $expectedStop) { foreach ($handles as $handle) { - if (microtime(true) >= $cleanupDeadline - 2) { + if (microtime(true) >= $cleanupDeadline - 5) { $trace->record(['kind' => 'cleanup_error']); $failure = new RuntimeException('completion_workflow_cleanup_deadline', previous: $failure); break; @@ -263,6 +266,10 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, $description = $cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId); if (!in_array($description->status, ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { $cleanup->terminateWorkflow($handle->workflowId, 'Completion qualification cleanup', $handle->selectedRunId); + if (!in_array($cleanup->describeWorkflow($handle->workflowId, $handle->selectedRunId)->status, + ['completed', 'failed', 'cancelled', 'terminated', 'timed_out'], true)) { + throw new RuntimeException('completion_workflow_cleanup_unconfirmed'); + } } } catch (Throwable $cleanupError) { if ($cleanupError instanceof ServerException && $cleanupError->status === 404 && $handle->selectedRunId === null) continue; @@ -283,7 +290,8 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, return ['scenario' => $data['scenario'], 'outcome' => CompletionPollFailure::diagnostic($failure)['outcome'], 'diagnostic' => CompletionPollFailure::diagnostic($failure), 'events' => $trace->state()['events']]; } - if (($cleanup->listWorkers($queue)['workers'] ?? []) !== []) { + $workers = $cleanup->listWorkers($queue)['workers'] ?? null; + if (!is_array($workers) || $workers !== []) { throw new RuntimeException('completion_worker_registration_survived'); } $events = $trace->state()['events']; @@ -320,7 +328,10 @@ function completionScenario(CompletionPollSubject $subject, string $clientToken, if (!is_string($clientToken) || $clientToken === '' || !is_string($workerToken) || $workerToken === '' || $clientToken === $workerToken) { throw new RuntimeException('completion_scoped_test_tokens'); } - if ($subject->data['scenario'] === 'opposite-slots') { + if (in_array($subject->data['scenario'], CompletionPollLoadProfile::SCENARIOS, true)) { + require_once __DIR__.'/Support/CompletionPollLoad.php'; + $result = CompletionPollLoad::run($subject, $clientToken, $workerToken); + } elseif (in_array($subject->data['scenario'], ['opposite-slots', 'workflow-refence'], true)) { require_once __DIR__.'/Support/CompletionPollOppositeSlots.php'; $result = CompletionPollOppositeSlots::run($subject, $clientToken, $workerToken); } else { diff --git a/tests/CompletionPollConsumerLoadTest.php b/tests/CompletionPollConsumerLoadTest.php new file mode 100644 index 00000000..bdfbe669 --- /dev/null +++ b/tests/CompletionPollConsumerLoadTest.php @@ -0,0 +1,174 @@ + 1000, 'step_callbacks' => 1000, 'workflow_callbacks' => 200], CompletionPollLoadEvidence::counts($reports)); + $reports[1]['steps']['1:0'] = 1; + $this->expectExceptionMessage('completion_load_functional_counts'); + CompletionPollLoadEvidence::counts($reports); + } + + public function testMissingWorkCannotBeHiddenByACompletedWorkflowCount(): void + { + $reports = self::reports(); + unset($reports[0]['steps']['1:0']); + $this->expectExceptionMessage('completion_load_functional_counts'); + CompletionPollLoadEvidence::counts($reports); + } + + public function testAWorkerFailureCannotQualifyTiming(): void + { + $reports = self::reports(); + $reports[5]['outcome'] = 'runner-blocked'; + $this->expectExceptionMessage('completion_load_worker_failed'); + CompletionPollLoadEvidence::counts($reports); + } + + public function testComparisonPreservesSlowRepetitionsAndReportsRegression(): void + { + $comparison = CompletionPollLoadEvidence::compare(self::results()); + self::assertSame([11, 12, 13], $comparison['elapsed_seconds']['off']); + self::assertSame([21, 22, 23], $comparison['elapsed_seconds']['on']); + self::assertTrue($comparison['on_slower_than_off']); + self::assertSame('not_assessed', $comparison['performance_acceptance']); + self::assertEqualsWithDelta(12 / 22, $comparison['off_over_on_ratio'], 0.00001); + } + + #[DataProvider('invalidResults')] + public function testComparisonRefusesMixedIncompleteOrReusedCells(string $mutation, string $reason): void + { + $results = self::results(); + match ($mutation) { + 'database' => $results[5]['result']['load']['database_identity'] = $results[0]['result']['load']['database_identity'], + 'subject' => $results[5]['identity']['subject_sha256'] = $results[0]['identity']['subject_sha256'], + 'conditions' => $results[5]['result']['load']['conditions_sha256'] = str_repeat('e', 64), + 'image' => $results[5]['caller_server_image'] = 'ghcr.io/durable-workflow/server@sha256:'.str_repeat('f', 64), + 'cleanup' => $results[5]['result']['cleanup']['complete'] = false, + 'option' => $results[5]['result']['enable_completion_pipelining'] = false, + 'repetition' => $results[5]['result']['load']['repetition'] = 1, + 'incomplete' => array_pop($results), + }; + $this->expectExceptionMessage($reason); + CompletionPollLoadEvidence::compare($results); + } + + public static function invalidResults(): iterable + { + yield ['database', 'completion_load_reused_cell']; + yield ['subject', 'completion_load_reused_cell']; + yield ['conditions', 'completion_load_mixed_bindings']; + yield ['image', 'completion_load_mixed_bindings']; + yield ['cleanup', 'completion_load_result_not_qualified']; + yield ['option', 'completion_load_result_not_qualified']; + yield ['repetition', 'completion_load_duplicate_repetition']; + yield ['incomplete', 'completion_load_six_results_required']; + } + + public function testResultReadBindsTheParsedBytesToTheCallerDigest(): void + { + $path = tempnam(sys_get_temp_dir(), 'load-evidence-'); + $bytes = json_encode(self::results()[0], JSON_THROW_ON_ERROR); + file_put_contents($path, $bytes); + try { + self::assertSame(self::results()[0], CompletionPollLoadEvidence::read($path, hash('sha256', $bytes))); + file_put_contents($path, str_replace('11', '99', $bytes)); + $this->expectExceptionMessage('completion_load_result_identity'); + CompletionPollLoadEvidence::read($path, hash('sha256', $bytes)); + } finally { + unlink($path); + } + } + + public function testWorkerShutdownUsesOneGlobalGraceAndReapsEveryOwnedChild(): void + { + if (!function_exists('pcntl_fork') || !function_exists('posix_setsid')) $this->markTestSkipped('Requires Unix process groups.'); + $directory = sys_get_temp_dir().'/load-stop-'.bin2hex(random_bytes(6)); + mkdir($directory, 0700); + $pids = []; + try { + for ($i = 0; $i < 6; ++$i) { + $pid = pcntl_fork(); + self::assertNotSame(-1, $pid); + if ($pid === 0) { + if (posix_setsid() === -1) exit(2); + pcntl_signal(SIGTERM, SIG_IGN); + file_put_contents($directory.'/'.$i, 'ready'); + while (true) usleep(10_000); + } + $pids[] = $pid; + } + $until = hrtime(true) / 1e9 + 2; + while (count(glob($directory.'/*')) !== 6) { + if (hrtime(true) / 1e9 >= $until) self::fail('Child fixture readiness failed.'); + usleep(1000); + } + $started = hrtime(true); + try { + CompletionPollLoad::stop($pids, 0.1); + self::fail('Forced shutdown cannot pass cleanup.'); + } catch (RuntimeException $error) { + self::assertSame('completion_load_worker_stop_failed', $error->getMessage()); + } + self::assertLessThan(0.5, (hrtime(true) - $started) / 1e9); + foreach ($pids as $pid) { + self::assertSame(-1, pcntl_waitpid($pid, $status, WNOHANG)); + self::assertFalse(posix_kill(-$pid, 0)); + } + } finally { + foreach ($pids as $pid) { + posix_kill(-$pid, SIGKILL); + posix_kill($pid, SIGKILL); + pcntl_waitpid($pid, $status, WNOHANG); + } + foreach (glob($directory.'/*') as $file) unlink($file); + rmdir($directory); + } + } + + private static function reports(): array + { + $reports = array_fill(0, 6, ['outcome' => 'pass', 'workflow_callbacks' => 0, 'steps' => []]); + for ($index = 1; $index <= 200; ++$index) { + $worker = ($index - 1) % 6; + ++$reports[$worker]['workflow_callbacks']; + for ($step = 0; $step < 5; ++$step) $reports[$worker]['steps'][$index.':'.$step] = 1; + } + return $reports; + } + + private static function results(): array + { + $results = []; + foreach (['off', 'on'] as $mode) { + for ($repetition = 1; $repetition <= 3; ++$repetition) { + $scenario = 'load-local-'.$mode; + $results[] = ['schema' => 'durable-workflow.php-completion-polls-result/v1', 'outcome' => 'pass', + 'identity' => ['sdk_version' => '9.0.0', 'sdk_reference' => str_repeat('a', 40), 'caller_runner_reference' => str_repeat('b', 40), + 'lock_sha256' => str_repeat('c', 64), 'subject_sha256' => hash('sha256', $mode.$repetition)], + 'server_version' => '9.0.0', 'caller_server_image' => 'ghcr.io/durable-workflow/server@sha256:'.str_repeat('d', 64), + 'result' => ['scenario' => $scenario, 'outcome' => 'pass', 'workload' => CompletionPollLoadProfile::workload($scenario), + 'load' => ['backend' => 'sqlite', 'database_identity' => 'fresh-'.$mode.'-'.$repetition, 'repetition' => $repetition, + 'conditions_sha256' => str_repeat('a', 64)], 'enable_completion_pipelining' => $mode === 'on', + 'relay_used' => false, 'request_trace_used' => false, 'elapsed_seconds' => ($mode === 'on' ? 20 : 10) + $repetition, + 'workflow_results_verified' => 200, 'counts' => ['unique_steps' => 1000, 'step_callbacks' => 1000], 'cleanup' => ['complete' => true]]]; + } + } + return $results; + } +} diff --git a/tests/CompletionPollConsumerSubjectTest.php b/tests/CompletionPollConsumerSubjectTest.php index dbd63746..fc1cb3be 100644 --- a/tests/CompletionPollConsumerSubjectTest.php +++ b/tests/CompletionPollConsumerSubjectTest.php @@ -51,6 +51,17 @@ public function testPreflightReadsTheCallerPacketWithoutLoadingConsumerCode(): v self::assertSame(hash_file('sha256', $this->directory.'/subject.json'), $subject->sha256); } + public function testLoadSelectionRequiresClosedCallerDatabaseAndConditionsBindings(): void + { + $this->data['scenario'] = 'load-remote-on'; + $this->data['load'] = ['backend' => 'postgresql', 'database_identity' => 'fresh-source-fixture', + 'repetition' => 1, 'conditions_sha256' => str_repeat('a', 64)]; + self::assertSame($this->data, $this->read()->data); + $this->data['load']['reset_database'] = true; + $this->expectExceptionMessage('completion_subject_load'); + $this->read(); + } + #[DataProvider('replacedDocuments')] public function testPathReplacementCannotChangeTheAuthenticatedDocument(string $document): void { diff --git a/tests/CompletionPollConsumerWorkflowRefenceTest.php b/tests/CompletionPollConsumerWorkflowRefenceTest.php new file mode 100644 index 00000000..9c7f88c4 --- /dev/null +++ b/tests/CompletionPollConsumerWorkflowRefenceTest.php @@ -0,0 +1,62 @@ + 2])); + $this->addToAssertionCount(1); + } + + #[DataProvider('changedRecovery')] + public function testRecoveryCannotReuseTheAttemptOrChangeHistory(string $field, mixed $value): void + { + $this->expectExceptionMessage('completion_refence_recovery_changed'); + CompletionPollWorkflowRefence::recovered(self::task(), array_replace(self::task(), [$field => $value])); + } + + public static function changedRecovery(): iterable + { + yield ['workflow_task_attempt', 1]; + yield ['task_id', 'different-task']; + yield ['history_events', [['type' => 'invented_repair']]]; + } + + public function testRefusalPrecedenceIsBoundToTheOriginalTaskAndCurrentAttempt(): void + { + $before = new ServerException('fixture', 409, 'task_not_leased', + ['task_id' => 'task', 'workflow_task_attempt' => 1, 'task_status' => 'ready', 'recorded' => false]); + CompletionPollWorkflowRefence::refusal($before, self::task(), 'task_not_leased', null, true); + $after = new ServerException('fixture', 409, 'workflow_task_attempt_mismatch', + ['task_id' => 'task', 'workflow_task_attempt' => 1, 'current_attempt' => 2]); + CompletionPollWorkflowRefence::refusal($after, self::task(), 'workflow_task_attempt_mismatch', 2, true); + $this->addToAssertionCount(2); + $this->expectExceptionMessage('completion_refence_wrong_refusal'); + CompletionPollWorkflowRefence::refusal($before, self::task(), 'workflow_task_attempt_mismatch', 2, true); + } + + public function testAnUnrelatedFailureCannotStandInForRevokedAuthority(): void + { + $this->expectExceptionMessage('completion_refence_wrong_refusal'); + CompletionPollWorkflowRefence::refusal(new ServerException('fixture', 503, 'backend_unavailable'), self::task(), 'task_not_leased', null, false); + } + + private static function task(): array + { + return ['task_id' => 'task', 'workflow_id' => 'workflow', 'run_id' => 'run', 'workflow_type' => 'qualification.wire', + 'lease_owner' => 'worker', 'workflow_task_attempt' => 1, 'lease_expires_at' => '2026-11-07T08:00:30.000000Z', + 'input' => ['synthetic' => true], 'history_events' => [['type' => 'activity_completed']]]; + } +}