From 5a4c3f7a9355ca1f94da25d60c2b86280f7675bc Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:51:50 +0000 Subject: [PATCH 01/15] Draft native retry policy adapter vectors Refs durable-workflow/.github#219. Bind canonical v2 policy IDs to native transport, time, attempt and acknowledgement requirements without changing a producer. --- .../native-adapters.v1.json | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json diff --git a/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json new file mode 100644 index 000000000..287ea6809 --- /dev/null +++ b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json @@ -0,0 +1,188 @@ +{ + "schema": "durable-workflow.native-activity-retry-adapters.v1", + "status": "source proposal; no executor, producer or qualification result", + "runtime_enabled": false, + "canonical": { + "repository": "durable-workflow/workflow", + "commit": "0aa1536e6198a6667ddedf62b48215b6e9e8a6f8", + "path": "resources/protocol/activity-retry-policy-v2-vectors.json", + "sha256": "8d576f1d50bcbfb123cb09682a4fbac53cb018a3cd369021c38d69866849781c", + "case_count": 54, + "mutate_canonical_rows": false + }, + "authority": { + "decision": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6643667500", + "native_bindings": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6647329559", + "native_reference_commit": "ea993064428fa9e0294520aa3efd498bef5b64dc", + "capability": "activity_retry_delay_override_v1", + "failure_override_field": "failure.next_retry_delay_seconds" + }, + "executor_requirements": { + "context": "otherwise valid synthetic authorized activity failure; original task/run/execution/attempt identity and complete before/after durable snapshot", + "carrier": "the actual native HTTP handler and authoritative failure transaction; no scenario-only decision model", + "raw_lexemes": "splice only the declared synthetic lexeme into an otherwise valid request; these strings are test bytes, not JSON numeric fixture values", + "typed_cases": "also exercise every canonical invalid_override_* case as its original actual JSON type", + "time": "one fixed authoritative failure sample retaining subsecond precision; each supported backend's checked bindable timestamp range", + "snapshots": "task/attempt/activity/history/receipt state, original deadlines and retry-task/event counts; no raw payload, credential or provider error output", + "retention": "compare the committed engine disposition and original availability after uncertainty; never recompute at replay time", + "result_scope": "an absent executor, unavailable backend or unimplemented version-2 producer is unqualified, never pass" + }, + "cases": [ + { + "id": "native_transport_nan", + "canonical_ids": ["invalid_override_integral_float"], + "carrier": {"override_json_lexeme": "NaN", "content_type": "application/json"}, + "expected": {"http_status": 400, "classification": "existing-json-syntax-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_transport_positive_infinity", + "canonical_ids": ["invalid_override_fractional_float"], + "carrier": {"override_json_lexeme": "Infinity", "content_type": "application/json"}, + "expected": {"http_status": 400, "classification": "existing-json-syntax-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_transport_negative_infinity", + "canonical_ids": ["invalid_override_negative"], + "carrier": {"override_json_lexeme": "-Infinity", "content_type": "application/json"}, + "expected": {"http_status": 400, "classification": "existing-json-syntax-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_transport_overflowing_exponent", + "canonical_ids": ["invalid_override_overflow"], + "carrier": {"override_json_lexeme": "1e9999", "content_type": "application/json"}, + "expected": {"http_status": 400, "classification": "existing-json-syntax-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_transport_trailing_document_bytes", + "canonical_ids": ["zero_override_is_present"], + "carrier": {"override_json_lexeme": "0", "append_after_document_utf8": " trailing", "content_type": "application/json"}, + "expected": {"http_status": 400, "classification": "existing-json-syntax-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_transport_missing_json_content_type", + "canonical_ids": ["zero_override_is_present"], + "carrier": {"override_json_lexeme": "0", "omit_content_type": true}, + "expected": {"http_status": 415, "classification": "existing-json-content-type-refusal", "durable_write_count": 0, "typed_override_verdict": false} + }, + { + "id": "native_typed_exponent_is_float", + "canonical_ids": ["invalid_override_integral_float"], + "carrier": {"override_json_lexeme": "1e0", "content_type": "application/json"}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "coercion": false} + }, + { + "id": "native_typed_nan_string", + "canonical_ids": ["invalid_override_string"], + "carrier": {"override_json_lexeme": "\"NaN\"", "content_type": "application/json"}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "transport_parse_failure": false} + }, + { + "id": "native_json_bound_is_not_timestamp_bound", + "canonical_ids": ["invalid_override_overflow"], + "carrier": {"override_json_lexeme": "9007199254740991", "failure_sample": "fixed-valid-native-timestamp", "precondition": "duration-or-resulting-timestamp-not-representable"}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "wrap_clamp_or_immediate_retry": false} + }, + { + "id": "native_backend_timestamp_addition_overflow", + "canonical_ids": ["invalid_override_overflow"], + "carrier": {"override_json_lexeme": "1", "failure_sample": "maximum-bindable-backend-timestamp", "backends": ["sqlite", "postgresql", "mysql", "mariadb"]}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "backend_error_counted_as_validation_pass": false} + }, + { + "id": "native_subsecond_failure_sample_preserved", + "canonical_ids": ["override_replaces_policy", "override_before_deadline"], + "carrier": {"failure_sample_subsecond_microseconds": 123456, "override_json_lexeme": "5", "original_deadline": "strictly-after-resulting-availability"}, + "expected": {"availability_adds_exact_whole_seconds": 5, "availability_subsecond_microseconds": 123456, "serialization_clock_resample": false, "original_deadline_unchanged": true} + }, + { + "id": "native_exact_subsecond_deadline_boundary", + "canonical_ids": ["override_reaches_deadline", "override_passes_deadline", "zero_at_original_deadline_is_terminal"], + "carrier": {"failure_sample_subsecond_microseconds": 123456, "deadline_cases": ["availability-equals-original-deadline", "availability-after-original-deadline"]}, + "expected": {"decision": "terminal", "reason": "retry_delay_exceeds_schedule_to_close", "retry_task_count": 0, "activity_retry_scheduled_count": 0, "round_to_seconds": false, "original_deadline_unchanged": true} + }, + { + "id": "native_activity_attempt_belongs_to_original_task", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"mutation": "submit-another-tasks-activity-attempt-id"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "durable_write_count": 0, "foreign_attempt_or_task_mutated": false} + }, + { + "id": "native_attempt_number_matches_task_fence", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"mutation": "stored-attempt-number-differs-from-original-task-attempt-count"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "durable_write_count": 0} + }, + { + "id": "native_wrong_lease_owner", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"mutation": "otherwise-current-failure-with-another-synthetic-owner"}, + "expected": {"http_status": 409, "reason": "lease_owner_mismatch", "durable_write_count": 0} + }, + { + "id": "native_expired_original_lease", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"mutation": "otherwise-current-failure-at-or-after-original-lease-expiry"}, + "expected": {"http_status": 409, "reason": "lease_expired", "durable_write_count": 0, "deadline_extension": false} + }, + { + "id": "native_cross_namespace_cannot_reach_original_task", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"mutation": "submit-original-task-and-attempt-in-another-authorized-synthetic-namespace"}, + "expected": {"classification": "existing-native-namespace-task-refusal", "durable_write_count": 0, "either_namespace_mutated": false} + }, + { + "id": "native_registration_replacement_revokes_original_claim", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"actions": ["claim-original-attempt", "commit-worker-registration-replacement", "submit-old-original-attempt"]}, + "expected": {"classification": "existing-native-attempt-or-lease-fence-refusal", "old_claim_accepted": false, "durable_write_count_from_old_report": 0, "replacement_authority_changed": false} + }, + { + "id": "native_two_connections_same_original_attempt", + "canonical_ids": ["duplicate_failure_adds_no_attempt"], + "carrier": {"actions": ["barrier-two-independent-connections", "report-same-current-attempt-with-different-valid-overrides"]}, + "expected": {"maximum_committed_failure_outcomes": 1, "maximum_additional_retry_tasks": 1, "mixed_disposition_fields": false, "loser_preserves_existing_native_fence_route": true} + }, + { + "id": "native_duplicate_keeps_stale_acknowledgement", + "canonical_ids": ["duplicate_failure_adds_no_attempt"], + "carrier": {"actions": ["commit-original-failure", "repeat-identical-failure"]}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "additional_retry_task_count": 0, "retained_disposition_unchanged": true} + }, + { + "id": "native_lost_ack_cold_replay_keeps_original_availability", + "canonical_ids": ["lost_acknowledgement_cold_retry", "version_2_cold_replay_uses_retained_outcome"], + "carrier": {"actions": ["commit-original-failure", "drop-http-acknowledgement", "replace-process-with-same-durable-state", "repeat-identical-failure-at-later-clock"]}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true, "recompute_policy_or_clock": false} + }, + { + "id": "native_conflicting_duplicate_cannot_rewrite_delay", + "canonical_ids": ["duplicate_failure_adds_no_attempt", "lost_acknowledgement_cold_retry"], + "carrier": {"actions": ["commit-original-failure", "retry-same-original-attempt-with-different-valid-delay"]}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true} + }, + { + "id": "native_scheduled_rows_and_event_agree", + "canonical_ids": ["zero_override_is_present", "override_replaces_policy", "override_at_cap", "empty_explicit_cap_accepts_override", "derived_cap_uses_largest_entry"], + "carrier": {"actions": ["submit-canonical-current-failure", "read-original-attempt-retry-task-event-and-engine-disposition"]}, + "expected": {"retry_task_count": 1, "activity_retry_scheduled_count": 1, "selected_delay_and_original_attempt_identity_match_everywhere": true, "original_failure_preserved": true, "original_deadline_unchanged": true} + }, + { + "id": "native_terminal_disposition_is_separate_from_application_failure", + "canonical_ids": ["override_exceeds_cap", "empty_default_cap_refuses_positive_override", "policy_reaches_deadline", "zero_at_original_deadline_is_terminal"], + "carrier": {"actions": ["submit-canonical-current-failure", "read-terminal-failure-and-workflow-resume-state"]}, + "expected": {"retry_task_count": 0, "activity_retry_scheduled_count": 0, "engine_reason_replaces_application_exception": false, "original_failure_preserved": true, "original_deadline_unchanged": true, "ordinary_terminal_resume_path": true} + }, + { + "id": "native_capability_and_snapshot_are_independent_gates", + "canonical_ids": ["missing_capability", "nonliteral_capability_false", "nonliteral_capability_string", "nonliteral_capability_integer", "legacy_snapshot_cannot_accept_override"], + "carrier": {"actions": ["submit-valid-present-override-with-each-canonical-gate-mutation"]}, + "expected": {"literal_capability_and_snapshot_2_both_required": true, "durable_write_count": 0, "silent_ignore_or_implicit_upgrade": false} + }, + { + "id": "native_legacy_and_recorded_availability_not_reinterpreted", + "canonical_ids": ["legacy_snapshot_cannot_accept_override", "core_snapshot_1_repeats_last", "core_snapshot_1_explicit_empty", "core_snapshot_1_no_generated_cap", "legacy_sdk_recorded_zero_tail"], + "carrier": {"actions": ["load-original-legacy-snapshot-and-recorded-outcomes", "cold-replay-at-later-clock"]}, + "expected": {"version_2_defaults_applied": false, "recorded_zero_tail_preserved": true, "original_availability_unchanged": true, "implicit_snapshot_upgrade": false} + } + ] +} From 6a45342ad51ac1a82d78db770b875faddeedc4ba Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:56:13 +0000 Subject: [PATCH 02/15] Keep a valid lease in the retry timestamp overflow vector --- .../contracts/activity-retry-policy/native-adapters.v1.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json index 287ea6809..431582923 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json +++ b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json @@ -85,7 +85,7 @@ { "id": "native_backend_timestamp_addition_overflow", "canonical_ids": ["invalid_override_overflow"], - "carrier": {"override_json_lexeme": "1", "failure_sample": "maximum-bindable-backend-timestamp", "backends": ["sqlite", "postgresql", "mysql", "mariadb"]}, + "carrier": {"override_json_lexeme": "1", "failure_sample": "maximum-bindable-backend-timestamp-minus-500000-microseconds", "lease_expires_at": "maximum-bindable-backend-timestamp", "original_schedule_to_close_seconds": null, "backends": ["sqlite", "postgresql", "mysql", "mariadb"]}, "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "backend_error_counted_as_validation_pass": false} }, { From 9bc440bf222f9cfc9e4401a3ce3b67246bf558de Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 02:41:26 +0000 Subject: [PATCH 03/15] test: bind native retry refusal and identity vectors --- .../native-adapters.v1.json | 195 ++++++++++++++++-- 1 file changed, 174 insertions(+), 21 deletions(-) diff --git a/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json index 431582923..795a1cdde 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json +++ b/tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json @@ -27,6 +27,93 @@ "retention": "compare the committed engine disposition and original availability after uncertainty; never recompute at replay time", "result_scope": "an absent executor, unavailable backend or unimplemented version-2 producer is unqualified, never pass" }, + "response_contracts": { + "bindings": { + "$reason": "the case expected.reason, or its explicitly named expected.loser.reason", + "$task_id": "the submitted HTTP path task ID; not a looked-up foreign identity", + "$activity_attempt_id": "the original submitted body.activity_attempt_id", + "$protocol_version": "exact version in the independently pinned source-under-test worker manifest; native reference is 1.20", + "$server_capabilities": "the entire independently pinned source-under-test capability manifest, deep exact equality; do not learn expected values from the failure response" + }, + "matching": "Resolve only the declared binding strings; compare the complete JSON object with exact keys and actual JSON types. No extra/missing field or arbitrary failure status passes. The worker protocol response header must equal the independently pinned protocol version.", + "reason_message_refusal": { + "body": { + "reason": "$reason", + "message": "$reason", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "recorded_member": "absent, not false or null", + "next_task_id_member": "absent, not null" + }, + "activity_task_not_found": { + "body": { + "task_id": "$task_id", + "activity_attempt_id": "$activity_attempt_id", + "error": "Activity task not found.", + "reason": "task_not_found", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "recorded_member": "absent, not false or null", + "next_task_id_member": "absent, not null" + }, + "stale_activity_acknowledgement": { + "body": { + "task_id": "$task_id", + "activity_attempt_id": "$activity_attempt_id", + "outcome": "failed", + "recorded": false, + "reason": "stale_attempt", + "next_task_id": null, + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + } + } + }, + "identity_isolation": { + "scope": "Future isolated source-test fixture seeding only; caller run/execution fields do not establish authority. Ignored request fields are not a mismatch control.", + "baseline": { + "namespace": "S", + "task": "T", + "run": "R", + "activity_execution": "E", + "activity_attempt": "A", + "lease_owner": "W", + "task_attempt_count": 1, + "activity_attempt_number": 1, + "activity_execution_attempt_count": 1, + "current_attempt_id": "A", + "max_attempts": 3, + "task_status": "leased", + "attempt_status": "running", + "activity_status": "running", + "run_status": "running", + "lease_and_original_deadlines": "strictly after the fixed failure sample" + }, + "alternate_rows": "T2/A2/E2 form an independently valid current activity in the same R and S, also leased to W with count 1; R2 is another active run in S. W2 is another registered synthetic owner. All alternate rows exist and are readable.", + "rule": "Start each row from a fresh valid baseline. Apply only the named request or durable-edge mutation before the before-snapshot. Every unlisted axis, auth/protocol gate, valid override and complete failure remains at baseline. Compare all original and alternate rows after refusal.", + "positive_control": "The unmutated baseline must reach the actual version-2 failure path and record its selected retry. Setup/constraint failure, absent producer, backend/codec/internal error or a refusal on another axis is unqualified, never this negative passing.", + "response_source": { + "prepare_order": "rust/src/runtime/execution.rs::Runtime::fail_activity -> prepare_failure -> backend fail_activity", + "attempt_order": "rust/src/runtime/activity_failures.rs::Store::fail_activity", + "namespace_lookup": "rust/src/runtime/store.rs::task_row", + "replacement": "rust/src/runtime/worker_management.rs::release_replaced_activity_leases", + "http_envelope": "rust/src/runtime/http.rs::activity_not_found and worker_protocol_envelope; rust/src/runtime/mod.rs::RuntimeError::into_response" + }, + "required_source_boundary": "These are required version-2 adoption controls, not an executing result or a claim that reference v1 implements each cross-graph guard. Return any missing task/run/execution consistency check in the future v2 source review; generic backend refusal never passes." + }, + "refusal_precedence": { + "order": [ + "existing authentication/namespace authorization and protocol admission", + "JSON extraction", + "strict supplied-override JSON type and portable integer range validation in failure preparation, before storage authority lookup", + "original namespace/task/attempt/lease/deadline authority", + "non-retryable then exhausted-attempt terminal classification, then cap/deadline retry selection" + ], + "scope": "This selects input-before-storage-authority for a malformed supplied override after successful auth/protocol admission. The new strict override validation is a required future source change, not behavior implemented by this data.", + "non_disclosure": "The same malformed valid-JSON override returns the same closed 422 body for an owned stale task and a foreign/missing task in another authorized namespace. Do not query or echo task existence, foreign run/execution identities or terminal state before that refusal." + }, "cases": [ { "id": "native_transport_nan", @@ -68,25 +155,25 @@ "id": "native_typed_exponent_is_float", "canonical_ids": ["invalid_override_integral_float"], "carrier": {"override_json_lexeme": "1e0", "content_type": "application/json"}, - "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "coercion": false} + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "coercion": false, "response_contract": "reason_message_refusal"} }, { "id": "native_typed_nan_string", "canonical_ids": ["invalid_override_string"], "carrier": {"override_json_lexeme": "\"NaN\"", "content_type": "application/json"}, - "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "transport_parse_failure": false} + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "transport_parse_failure": false, "response_contract": "reason_message_refusal"} }, { "id": "native_json_bound_is_not_timestamp_bound", "canonical_ids": ["invalid_override_overflow"], "carrier": {"override_json_lexeme": "9007199254740991", "failure_sample": "fixed-valid-native-timestamp", "precondition": "duration-or-resulting-timestamp-not-representable"}, - "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "wrap_clamp_or_immediate_retry": false} + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "wrap_clamp_or_immediate_retry": false, "response_contract": "reason_message_refusal"} }, { "id": "native_backend_timestamp_addition_overflow", "canonical_ids": ["invalid_override_overflow"], "carrier": {"override_json_lexeme": "1", "failure_sample": "maximum-bindable-backend-timestamp-minus-500000-microseconds", "lease_expires_at": "maximum-bindable-backend-timestamp", "original_schedule_to_close_seconds": null, "backends": ["sqlite", "postgresql", "mysql", "mariadb"]}, - "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "backend_error_counted_as_validation_pass": false} + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "durable_write_count": 0, "backend_error_counted_as_validation_pass": false, "response_contract": "reason_message_refusal"} }, { "id": "native_subsecond_failure_sample_preserved", @@ -103,62 +190,62 @@ { "id": "native_activity_attempt_belongs_to_original_task", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"mutation": "submit-another-tasks-activity-attempt-id"}, - "expected": {"http_status": 409, "reason": "stale_attempt", "durable_write_count": 0, "foreign_attempt_or_task_mutated": false} + "carrier": {"mutation": "submit-another-tasks-activity-attempt-id", "identity_isolation": "required", "single_mutation": {"location": "request.body.activity_attempt_id", "replace_with": "A2"}}, + "expected": {"http_status": 409, "reason": "stale_attempt", "durable_write_count": 0, "foreign_attempt_or_task_mutated": false, "response_contract": "reason_message_refusal"} }, { "id": "native_attempt_number_matches_task_fence", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"mutation": "stored-attempt-number-differs-from-original-task-attempt-count"}, - "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "durable_write_count": 0} + "carrier": {"mutation": "stored-attempt-number-differs-from-original-task-attempt-count", "identity_isolation": "required", "single_mutation": {"location": "activity_attempts[A].attempt_number", "replace_with": 2}}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "durable_write_count": 0, "response_contract": "stale_activity_acknowledgement", "next_task_id": null} }, { "id": "native_wrong_lease_owner", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"mutation": "otherwise-current-failure-with-another-synthetic-owner"}, - "expected": {"http_status": 409, "reason": "lease_owner_mismatch", "durable_write_count": 0} + "carrier": {"mutation": "otherwise-current-failure-with-another-synthetic-owner", "identity_isolation": "required", "single_mutation": {"location": "request.body.lease_owner", "replace_with": "W2"}}, + "expected": {"http_status": 409, "reason": "lease_owner_mismatch", "durable_write_count": 0, "response_contract": "reason_message_refusal"} }, { "id": "native_expired_original_lease", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"mutation": "otherwise-current-failure-at-or-after-original-lease-expiry"}, - "expected": {"http_status": 409, "reason": "lease_expired", "durable_write_count": 0, "deadline_extension": false} + "carrier": {"mutation": "otherwise-current-failure-at-or-after-original-lease-expiry", "identity_isolation": "required", "single_mutation": {"location": "workflow_tasks[T].lease_expires_at", "replace_with": "fixed-authoritative-failure-sample"}}, + "expected": {"http_status": 409, "reason": "lease_expired", "durable_write_count": 0, "deadline_extension": false, "response_contract": "reason_message_refusal"} }, { "id": "native_cross_namespace_cannot_reach_original_task", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"mutation": "submit-original-task-and-attempt-in-another-authorized-synthetic-namespace"}, - "expected": {"classification": "existing-native-namespace-task-refusal", "durable_write_count": 0, "either_namespace_mutated": false} + "carrier": {"mutation": "submit-original-task-and-attempt-in-another-authorized-synthetic-namespace", "preconditions": "Valid override and failure, valid worker-plane auth/protocol for the other namespace, no same-ID task there; both namespace snapshots retained."}, + "expected": {"durable_write_count": 0, "either_namespace_mutated": false, "http_status": 404, "reason": "task_not_found", "response_contract": "activity_task_not_found", "authority_metadata_disclosed": false} }, { "id": "native_registration_replacement_revokes_original_claim", "canonical_ids": ["stale_attempt_cannot_schedule"], - "carrier": {"actions": ["claim-original-attempt", "commit-worker-registration-replacement", "submit-old-original-attempt"]}, - "expected": {"classification": "existing-native-attempt-or-lease-fence-refusal", "old_claim_accepted": false, "durable_write_count_from_old_report": 0, "replacement_authority_changed": false} + "carrier": {"actions": ["claim-original-attempt", "commit-worker-registration-replacement", "submit-old-original-attempt"], "preconditions": "Committed replacement expires A and returns T to ready through release_replaced_activity_leases; observe that state before the old report, with no re-poll, repair, cancellation or other maintenance."}, + "expected": {"old_claim_accepted": false, "durable_write_count_from_old_report": 0, "replacement_authority_changed": false, "http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "response_contract": "stale_activity_acknowledgement", "replacement_snapshot_unchanged_by_old_report": true} }, { "id": "native_two_connections_same_original_attempt", "canonical_ids": ["duplicate_failure_adds_no_attempt"], - "carrier": {"actions": ["barrier-two-independent-connections", "report-same-current-attempt-with-different-valid-overrides"]}, - "expected": {"maximum_committed_failure_outcomes": 1, "maximum_additional_retry_tasks": 1, "mixed_disposition_fields": false, "loser_preserves_existing_native_fence_route": true} + "carrier": {"actions": ["barrier-two-independent-connections", "report-same-current-attempt-with-different-valid-overrides"], "preconditions": "Both valid requests target the same current original attempt; barrier is before transaction acquisition, never inside a held serialization lock. The winner commits before the loser observes authoritative state; no maintenance/claim/cancellation interferes."}, + "expected": {"maximum_committed_failure_outcomes": 1, "maximum_additional_retry_tasks": 1, "mixed_disposition_fields": false, "loser": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "response_contract": "stale_activity_acknowledgement", "durable_write_count": 0, "winner_retained_snapshot_unchanged": true}, "committed_failure_outcome_count": 1, "additional_retry_task_count": 1} }, { "id": "native_duplicate_keeps_stale_acknowledgement", "canonical_ids": ["duplicate_failure_adds_no_attempt"], "carrier": {"actions": ["commit-original-failure", "repeat-identical-failure"]}, - "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "additional_retry_task_count": 0, "retained_disposition_unchanged": true} + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "response_contract": "stale_activity_acknowledgement"} }, { "id": "native_lost_ack_cold_replay_keeps_original_availability", "canonical_ids": ["lost_acknowledgement_cold_retry", "version_2_cold_replay_uses_retained_outcome"], "carrier": {"actions": ["commit-original-failure", "drop-http-acknowledgement", "replace-process-with-same-durable-state", "repeat-identical-failure-at-later-clock"]}, - "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true, "recompute_policy_or_clock": false} + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true, "recompute_policy_or_clock": false, "response_contract": "stale_activity_acknowledgement", "next_task_id": null} }, { "id": "native_conflicting_duplicate_cannot_rewrite_delay", "canonical_ids": ["duplicate_failure_adds_no_attempt", "lost_acknowledgement_cold_retry"], "carrier": {"actions": ["commit-original-failure", "retry-same-original-attempt-with-different-valid-delay"]}, - "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true} + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "additional_retry_task_count": 0, "retained_disposition_unchanged": true, "original_availability_unchanged": true, "response_contract": "stale_activity_acknowledgement", "next_task_id": null} }, { "id": "native_scheduled_rows_and_event_agree", @@ -183,6 +270,72 @@ "canonical_ids": ["legacy_snapshot_cannot_accept_override", "core_snapshot_1_repeats_last", "core_snapshot_1_explicit_empty", "core_snapshot_1_no_generated_cap", "legacy_sdk_recorded_zero_tail"], "carrier": {"actions": ["load-original-legacy-snapshot-and-recorded-outcomes", "cold-replay-at-later-clock"]}, "expected": {"version_2_defaults_applied": false, "recorded_zero_tail_preserved": true, "original_availability_unchanged": true, "implicit_snapshot_upgrade": false} + }, + { + "id": "native_malformed_override_precedes_stale_and_non_retryable", + "canonical_ids": ["invalid_override_null", "stale_attempt_cannot_schedule", "invalid_input_precedes_non_retryable"], + "carrier": {"override_json_lexeme": "null", "content_type": "application/json", "failure_fields": {"non_retryable": true}, "preconditions": "Successful auth/protocol in S; otherwise valid failure; original attempt finished and original count at max. Retain the complete committed original outcome before submitting.", "paired_with_case_id": "native_malformed_override_precedes_foreign_and_exhausted"}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "response_contract": "reason_message_refusal", "durable_write_count": 0, "task_authority_lookup_performed": false, "terminal_disposition_written": false, "foreign_identity_or_terminal_metadata_disclosed": false} + }, + { + "id": "native_malformed_override_precedes_foreign_and_exhausted", + "canonical_ids": ["invalid_override_null", "stale_attempt_cannot_schedule", "invalid_input_precedes_non_retryable", "exhausted_attempts_precede_cap"], + "carrier": {"override_json_lexeme": "null", "content_type": "application/json", "failure_fields": {"non_retryable": true}, "preconditions": "Use the paired case same original task/attempt IDs, identical malformed body and retained terminal context; only request namespace changes from S to authorized S2. Original task exists only in S and no same-ID task exists in S2.", "paired_with_case_id": "native_malformed_override_precedes_stale_and_non_retryable"}, + "expected": {"http_status": 422, "reason": "invalid_activity_retry_delay", "response_contract": "reason_message_refusal", "durable_write_count": 0, "task_authority_lookup_performed": false, "terminal_disposition_written": false, "foreign_identity_or_terminal_metadata_disclosed": false} + }, + { + "id": "native_valid_override_stale_precedes_terminal", + "canonical_ids": ["stale_attempt_cannot_schedule", "non_retryable_precedes_exhausted_attempts"], + "carrier": {"override_json_lexeme": "0", "failure_fields": {"non_retryable": true}, "preconditions": "Successful auth/protocol; original attempt finished, original count at max; retain that outcome before this stale report."}, + "expected": {"http_status": 409, "reason": "stale_attempt", "recorded": false, "next_task_id": null, "response_contract": "stale_activity_acknowledgement", "durable_write_count": 0, "terminal_disposition_written": false, "retained_outcome_unchanged": true} + }, + { + "id": "native_isolated_workflow_task_path_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "request.path.task_id", "replace_with": "T2", "body_activity_attempt_id": "A"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true} + }, + { + "id": "native_isolated_attempt_workflow_run_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "activity_attempts[A].workflow_run_id", "replace_with": "R2"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true} + }, + { + "id": "native_isolated_attempt_activity_execution_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "activity_attempts[A].activity_execution_id", "replace_with": "E2"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true} + }, + { + "id": "native_isolated_task_attempt_count_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "workflow_tasks[T].attempt_count", "replace_with": 2}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "stale_activity_acknowledgement", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true, "recorded": false, "next_task_id": null} + }, + { + "id": "native_isolated_activity_current_attempt_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "activity_executions[E].current_attempt_id", "replace_with": "A2"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true} + }, + { + "id": "native_isolated_activity_attempt_count_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "activity_executions[E].attempt_count", "replace_with": 2}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true} + }, + { + "id": "native_isolated_task_workflow_run_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "workflow_tasks[T].workflow_run_id", "replace_with": "R2"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true, "missing_reference_guard_requires_future_source_review": true} + }, + { + "id": "native_isolated_task_activity_execution_mismatch", + "canonical_ids": ["stale_attempt_cannot_schedule"], + "carrier": {"identity_isolation": "required", "single_mutation": {"location": "workflow_tasks[T].payload.activity_execution_id", "replace_with": "E2"}, "override_json_lexeme": "0"}, + "expected": {"http_status": 409, "reason": "stale_attempt", "response_contract": "reason_message_refusal", "durable_write_count": 0, "original_and_alternate_authority_unchanged": true, "missing_reference_guard_requires_future_source_review": true} } ] } From f3a5f95316c76853ff195edaa3445c5ccd96bfa5 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:00:20 +0000 Subject: [PATCH 04/15] Propose the prepared local retry decision companion --- docs/contracts/prepared-local-retry-v2.md | 64 + .../prepared-local.v2.json | 1034 +++++++++++++++++ 2 files changed, 1098 insertions(+) create mode 100644 docs/contracts/prepared-local-retry-v2.md create mode 100644 tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md new file mode 100644 index 000000000..f13cdeef5 --- /dev/null +++ b/docs/contracts/prepared-local-retry-v2.md @@ -0,0 +1,64 @@ +# Prepared local retry contract proposal + +This is an unfrozen source proposal for the local part of organization issue +[#219](https://github.com/durable-workflow/.github/issues/219). The machine-readable +companion is [prepared-local.v2.json](../../tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json). +It adds no runtime validator, producer, advertisement or executing test result. +The existing 54 canonical policy cases and 37 remote adapter cases are unchanged. + +Local version 2 requires the existing prepared per-attempt outcome authority, +the literal `activity_retry_delay_override_v1` capability on the original claim, +local contract 2 and retained snapshot 2 together. Current native Server lacks +the prepared role and truthfully refuses its routes. Its unprepared +`record_local_activity` batch remains version 1, including its zero-tail and +backdated history interpretation. A worker cannot run subsequent callbacks and +later use that batch to claim an engine-authorized version-2 decision. + +The prepared report has its own closed grammar. Only a failed report may add +`next_retry_delay_seconds`, as an actual nonnegative JSON integer within the +portable bound. Absence and zero differ. Completed and timed-out reports retain +their existing fields. Workers cannot supply a disposition, selected backoff, +wall-clock failure time, deadline, policy snapshot or engine timeout category. + +The engine uses one UTC sample under the canonical locks for fresh temporal +fences and retry selection. The first transaction that commits retains that +sample and decision. Rollback publishes neither. A matching outcome receipt +replays its stored decision before fresh retry calculation. The original total +deadline comes from the prepared execution and correlated start authority, +never a duplicate response's fresh `server_time` or a batch's recording clock. +The SDK must receive or recover that engine decision before sleeping or invoking +another callback. + +The proposed `retry_disposition` object has closed state-dependent fields in +the companion. Scheduled decisions retain candidate, cap and checked availability. +Terminal cap/deadline decisions retain a diagnostic candidate but have no retry +availability, task, start, following attempt or retry event. Non-retryable and +exhausted outcomes select no delay. An already-due prepared engine timeout +retains its engine provenance rather than borrowing the caller's application +failure. An earlier valid failure whose selected availability reaches the total +deadline remains an application failure with terminal retry-delay disposition. +General's reconciliation of that mode-bound projection is pending before freeze. + +The object is proposed engine-owned output shared by outcome history and the +matching receipt. It is not accepted worker input. Existing receipt identities, +claim-release behavior and created-task identity remain authoritative. The +companion closes this engine object, not the whole HTTP response. The controller's +history cursor, protocol envelope and exact refusal/status matrix still need a +separate source binding before adoption. No new top-level HTTP reason aliases +are selected here. + +The data's named source controls cover strict override types, separate report +grammars, conjunctive mode gates, isolated authoritative identities, deadline +edges, terminal precedence, reply loss, receipt mismatch, rollback, cold replay, +checked backend timestamp arithmetic and unchanged version-1 behavior. They are +requirements for a future real writer/adapter. Their presence or JSON consistency +is not an executing pass. The portable integer bound is distinct from an actual +backend duration or timestamp bound. Neither the old 100-report batch limit nor +its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. + +Workflow owns the complete normalized snapshot and existing timeout/range +binding. Server owns the companion validator, transaction, transport and storage +bindings. The exact output object, final normalized ranges, timeout projection +and HTTP bindings must be reconciled before these bytes are frozen or SDKs +adopt them. Native implementation, schema, installed/backend qualification and +release remain separate work. diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json new file mode 100644 index 000000000..99b45723d --- /dev/null +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -0,0 +1,1034 @@ +{ + "schema": "durable-workflow.prepared-local-retry-companion.v1", + "status": "unfrozen source proposal; no validator, writer or executing result", + "runtime_enabled": false, + "authority": { + "scope": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6668566058", + "core_clock": "https://github.com/durable-workflow/workflow/pull/1038#issuecomment-6669120859", + "native_role_inventory": "https://github.com/durable-workflow/server/pull/1034#issuecomment-6668962888", + "native_reference": "d8eb3bc1a5368752be823784521a9a09ec44de71", + "canonical": { + "commit": "0aa1536e6198a6667ddedf62b48215b6e9e8a6f8", + "path": "resources/protocol/activity-retry-policy-v2-vectors.json", + "sha256": "8d576f1d50bcbfb123cb09682a4fbac53cb018a3cd369021c38d69866849781c", + "case_count": 54 + }, + "remote_companion": { + "commit": "9bc440bf222f9cfc9e4401a3ce3b67246bf558de", + "path": "tests/Fixtures/contracts/activity-retry-policy/native-adapters.v1.json", + "sha256": "c37fb132129e4557c7ffbd90404763924640262bb5ed10e764f090f69ae12bf5", + "case_count": 37 + }, + "prior_rows_mutated": false + }, + "adoption_gates": [ + "General reconciles the prepared-timeout projection before final companion freeze", + "Core accepts the proposed closed retry_disposition fields and normalized snapshot/range binding", + "Server binds the full HTTP envelope/refusal matrix to the installed future prepared implementation", + "Native implements the prepared role and validator/transaction writer before advertisement", + "Actual native/database/SDK/installed qualification and release remain separate" + ], + "mode_gate": { + "conjunction": { + "installed_prepared_local_role": true, + "capability_activity_retry_delay_override_v1": true, + "local_contract_version": 2, + "retained_snapshot_version": 2, + "original_claim_supports_selected_local_v2_mode": true + }, + "literal_types": true, + "remote_capability_alone_suffices": false, + "unanchored_record_local_activity_batch": "version 1 only; never reinterpreted as an anchored version-2 result" + }, + "prepared_report": { + "boundary": "report object passed to the existing per-attempt recordLocalActivityOutcome flow; not the 11-field record_local_activity batch", + "unknown_members": "refuse before publication or a further callback", + "completed": { + "required": [ + "outcome", + "result" + ], + "optional": [ + "payload_codec" + ], + "outcome": "completed", + "result": "existing validated encoded Avro result; codec must resolve to avro" + }, + "failed": { + "required": [ + "outcome", + "message" + ], + "optional": [ + "exception_type", + "non_retryable", + "next_retry_delay_seconds" + ], + "outcome": "failed", + "message": "existing string", + "exception_type": "existing nonblank UTF-8 string, at most 255 bytes; existing RuntimeException default if absent", + "non_retryable": "Boolean only; existing false default if absent", + "next_retry_delay_seconds": "optional actual JSON integer in [0,9007199254740991]; null/Boolean/string/floating lexeme/refused" + }, + "timed_out": { + "required": [ + "outcome" + ], + "optional": [], + "outcome": "timed_out", + "classification": "engine-owned and actually due; no worker override, timeout kind, delay or failure timestamp" + }, + "worker_forbidden": [ + "retry_disposition", + "backoff_seconds", + "retry_backoff_seconds", + "retry_available_at", + "failure_time", + "original_schedule_to_close_deadline_at", + "snapshot_version" + ] + }, + "clock": { + "sample": "one engine UTC sample after strict request validation and canonical authority locks; the same sample drives fresh temporal fences and the retry decision", + "publication": "retained only with the first transaction that commits; rollback publishes neither a sample nor decision", + "duplicate": "return the matching committed per-attempt receipt and decision; never resample or rerun policy for that receipt", + "original_deadline": "stored prepared execution schedule_to_close_deadline_at, checked against correlated original preparation/start authority", + "not_anchors": [ + "duplicate prepare response server_time", + "batch commit time", + "worker absolute time", + "backdated version-1 batch timestamps" + ], + "serialization": "UTC RFC3339 with exactly six fractional digits and Z; nullable original schedule-to-close deadline", + "prepared_timeout_projection": "preserve existing engine timeout precedence at an already-due deadline; a valid earlier application failure whose selected availability reaches the deadline remains an application failure with terminal retry-delay disposition" + }, + "retry_disposition_proposal": { + "boundary": "engine-owned object shared exactly by the stored outcome metadata and its prepared receipt; never accepted in a worker report", + "common_required": { + "contract_version": 2, + "snapshot_version": 2, + "decision": "scheduled or terminal", + "failure_time": "the stored first accepted engine sample", + "original_schedule_to_close_deadline_at": "stored UTC timestamp or null" + }, + "common_optional": { + "requested_delay_seconds": "present iff the valid failed report supplied next_retry_delay_seconds; exact requested integer, including zero; diagnostic input does not prove application" + }, + "scheduled_required": { + "candidate_delay_seconds": "selected nonnegative portable integer", + "maximum_interval_seconds": "finite cap from retained snapshot 2", + "retry_available_at": "checked failure_time plus selected seconds, strictly before original deadline if set" + }, + "scheduled_forbidden": [ + "terminal_classification" + ], + "terminal_required": { + "terminal_classification": [ + "non_retryable", + "attempts_exhausted", + "retry_delay_exceeds_policy_cap", + "retry_delay_exceeds_schedule_to_close", + "engine_timeout" + ] + }, + "terminal_delay_classifications": [ + "retry_delay_exceeds_policy_cap", + "retry_delay_exceeds_schedule_to_close" + ], + "terminal_delay_required": [ + "candidate_delay_seconds", + "maximum_interval_seconds" + ], + "terminal_without_selection": [ + "non_retryable", + "attempts_exhausted", + "engine_timeout" + ], + "terminal_without_selection_forbidden": [ + "candidate_delay_seconds", + "maximum_interval_seconds" + ], + "terminal_forbidden": [ + "retry_available_at" + ], + "unknown_members": "refused by a future consumer; never silently ignored" + }, + "engine_receipt_proposal": { + "existing_required": [ + "recorded", + "duplicate", + "reason", + "event_id", + "event_type", + "workflow_run_id", + "workflow_task_id", + "workflow_task_attempt", + "activity_execution_id", + "activity_attempt_id", + "worker_attempt_id", + "recorded_at", + "claim_released", + "created_task_ids" + ], + "recorded": true, + "reason": null, + "new_optional": [ + "retry_disposition" + ], + "retry_disposition_present": "for a version-2 failed/timeout engine outcome; absent for a genuinely completed outcome", + "scheduled": { + "event_type": "ActivityRetryScheduled", + "claim_released": true, + "created_task_ids": "exactly one retained retry workflow task ID", + "retry_disposition": "scheduled" + }, + "terminal_failed": { + "event_type": "ActivityFailed", + "claim_released": false, + "created_task_ids": [], + "retry_disposition": "terminal" + }, + "terminal_timeout": { + "event_type": "ActivityTimedOut", + "claim_released": false, + "created_task_ids": [], + "retry_disposition": "terminal, with engine-owned timeout provenance" + }, + "completed": { + "event_type": "ActivityCompleted", + "claim_released": false, + "created_task_ids": [], + "retry_disposition": "absent" + }, + "http_boundary": "this closes the engine receipt object only; controller history_refresh_page_token, protocol envelope and refusal HTTP status/body require their own exact source binding before adoption" + }, + "history": { + "identity": "same run/execution/real attempt/worker attempt/original task epoch as the prepared outcome receipt", + "scheduled": "one ActivityRetryScheduled for the real failed attempt, with matching stored retry_disposition, retry_available_at and actual selected retry_backoff_seconds; no following attempt exists until independently prepared", + "terminal": "one existing ActivityFailed or ActivityTimedOut outcome for the real attempt; matching retry_disposition; no retry task, following attempt/start or ActivityRetryScheduled", + "failure_preservation": "cap/deadline retry termination preserves the admitted application failure/type; already-due engine timeout preserves engine timeout provenance instead", + "cold_replay": "consume the stored decision and availability; do not evaluate the current policy or clock" + }, + "ranges": { + "portable_integer_minimum": 0, + "portable_integer_maximum": 9007199254740991, + "legacy_batch_backoff_ceiling_seconds": 86400, + "legacy_batch_attempt_count_ceiling": 100, + "v2_inherits_these_batch_ceilings": false, + "checked_selection": "a selected retry must also fit the backend duration arithmetic and resulting bindable UTC timestamp before writes; no wrap, clamp, float coercion or immediate fallback", + "terminal_candidate": "a portable diagnostic integer is not an executed duration or retry availability", + "remaining_snapshot_binding": "Core must return exact policy/snapshot keys, finite max-attempt/list bounds and existing timeout field bounds; this proposal does not invent them from the batch limits" + }, + "cases": [ + { + "id": "local_v2_override_absent", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + } + }, + "expected": { + "report_shape": "accepted", + "requested_delay_present": false + }, + "canonical_ids": [ + "missing_override_uses_policy" + ] + }, + { + "id": "local_v2_override_zero", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 0 + } + }, + "expected": { + "report_shape": "accepted", + "requested_delay_present": true + }, + "canonical_ids": [ + "zero_override_is_present" + ] + }, + { + "id": "local_v2_override_positive", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 5 + } + }, + "expected": { + "report_shape": "accepted", + "requested_delay_present": true + }, + "canonical_ids": [ + "override_replaces_policy" + ] + }, + { + "id": "local_v2_override_null", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": null + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_true", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": true + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_false", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": false + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_string", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": "5" + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_fraction", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 1.5 + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_negative", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": -1 + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_portable_overflow", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 9007199254740992 + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_override_integral_float_lexeme", + "carrier": { + "raw_report_member": "\"next_retry_delay_seconds\":5.0" + }, + "expected": { + "report_shape": "refused", + "must_preserve_numeric_lexeme_class": true, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_completed_override_forbidden", + "carrier": { + "outcome": "completed", + "extra_member": { + "next_retry_delay_seconds": 0 + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_timed_out_override_forbidden", + "carrier": { + "outcome": "timed_out", + "extra_member": { + "next_retry_delay_seconds": 0 + } + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_retry_disposition_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "retry_disposition" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_backoff_seconds_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "backoff_seconds" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_retry_backoff_seconds_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "retry_backoff_seconds" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_retry_available_at_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "retry_available_at" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_failure_time_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "failure_time" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_original_schedule_to_close_deadline_at_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "original_schedule_to_close_deadline_at" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_worker_snapshot_version_forbidden", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false + }, + "extra_member": "snapshot_version" + }, + "expected": { + "report_shape": "refused", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_gate_installed_prepared_local_role", + "carrier": { + "otherwise_valid": true, + "one_missing_or_mismatching_gate": "installed_prepared_local_role" + }, + "expected": { + "local_v2": "unsupported", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_gate_capability_activity_retry_delay_override_v1", + "carrier": { + "otherwise_valid": true, + "one_missing_or_mismatching_gate": "capability_activity_retry_delay_override_v1" + }, + "expected": { + "local_v2": "unsupported", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_gate_local_contract_version", + "carrier": { + "otherwise_valid": true, + "one_missing_or_mismatching_gate": "local_contract_version" + }, + "expected": { + "local_v2": "unsupported", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_gate_retained_snapshot_version", + "carrier": { + "otherwise_valid": true, + "one_missing_or_mismatching_gate": "retained_snapshot_version" + }, + "expected": { + "local_v2": "unsupported", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_gate_original_claim_supports_selected_local_v2_mode", + "carrier": { + "otherwise_valid": true, + "one_missing_or_mismatching_gate": "original_claim_supports_selected_local_v2_mode" + }, + "expected": { + "local_v2": "unsupported", + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_run", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "run" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_execution", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "execution" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_activity_attempt", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "activity_attempt" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_worker_attempt", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "worker_attempt" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_workflow_task", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "workflow_task" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_task_epoch", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "task_epoch" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_owner", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "owner" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_descriptor_fingerprint", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "descriptor_fingerprint" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_original_start", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "original_start" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_authority_original_deadline", + "carrier": { + "otherwise_valid": true, + "isolated_authoritative_mismatch": "original_deadline" + }, + "expected": { + "recorded": false, + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_availability_before_deadline", + "carrier": { + "failure_time_seconds": 100, + "original_deadline_seconds": 105, + "override": 4, + "maximum_interval_seconds": 20, + "original_claim_live": true + }, + "expected": { + "decision": "scheduled", + "terminal_classification": null, + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "new_attempt_count": 0, + "original_application_failure_preserved": true + }, + "canonical_ids": [ + "override_before_deadline" + ] + }, + { + "id": "local_v2_availability_equal_deadline", + "carrier": { + "failure_time_seconds": 100, + "original_deadline_seconds": 105, + "override": 5, + "maximum_interval_seconds": 20, + "original_claim_live": true + }, + "expected": { + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "original_application_failure_preserved": true + }, + "canonical_ids": [ + "override_reaches_deadline" + ] + }, + { + "id": "local_v2_availability_after_deadline", + "carrier": { + "failure_time_seconds": 100, + "original_deadline_seconds": 105, + "override": 6, + "maximum_interval_seconds": 20, + "original_claim_live": true + }, + "expected": { + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "original_application_failure_preserved": true + }, + "canonical_ids": [ + "override_passes_deadline" + ] + }, + { + "id": "local_v2_at_deadline_engine_timeout", + "carrier": { + "engine_sample_seconds": 100, + "original_deadline_seconds": 100, + "submitted_report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 0 + }, + "lease_and_run_deadlines": "later than sample" + }, + "expected": { + "decision": "terminal", + "terminal_classification": "engine_timeout", + "override_applied": false, + "original_application_failure_preserved": false, + "engine_timeout_provenance_preserved": true, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "canonical_zero_at_deadline_is_prepared_http_pass": false + } + }, + { + "id": "local_v2_cap_exceeded", + "carrier": { + "original_claim_live": true, + "report": { + "outcome": "failed", + "message": "synthetic local failure", + "exception_type": "SyntheticFailure", + "non_retryable": false, + "next_retry_delay_seconds": 21 + }, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_policy_cap", + "candidate_delay_seconds": 21, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "original_application_failure_preserved": true + }, + "canonical_ids": [ + "override_exceeds_cap" + ] + }, + { + "id": "local_v2_precedence_non_retryable", + "carrier": { + "original_claim_live": true, + "one_terminal_axis": "non_retryable", + "valid_override": 21, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "terminal_classification": "non_retryable", + "candidate_delay_member": "absent", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_precedence_attempts_exhausted", + "carrier": { + "original_claim_live": true, + "one_terminal_axis": "attempts_exhausted", + "valid_override": 21, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "terminal_classification": "attempts_exhausted", + "candidate_delay_member": "absent", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_rollback", + "carrier": { + "first_transaction": "rollback" + }, + "expected": { + "published_sample_count": 0, + "published_decision_count": 0 + } + }, + { + "id": "local_v2_lost_outcome_reply", + "carrier": { + "first_outcome": "committed", + "reply": "lost", + "identical_retry_at": "later clock" + }, + "expected": { + "duplicate": true, + "stored_sample_and_decision_unchanged": true, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_changed_duplicate", + "carrier": { + "first_outcome": "committed", + "retry_override": "changed" + }, + "expected": { + "recorded": false, + "reason": "local_activity_outcome_mismatch", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_duplicate_after_expiry", + "carrier": { + "first_outcome": "committed", + "identical_retry": "after original lease expiry" + }, + "expected": { + "duplicate": true, + "stored_receipt_first": true, + "new_retry_calculation_count": 0 + } + }, + { + "id": "local_v2_missing_anchor", + "carrier": { + "execution": "v1 unprepared batch with otherwise plausible duration reports" + }, + "expected": { + "local_v2": "unsupported", + "anchor_reconstruction_count": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_duplicate_prepare_time", + "carrier": { + "original_prepare": "retained", + "duplicate_server_time": "later" + }, + "expected": { + "original_deadline_unchanged": true, + "new_failure_clock_authority": false + } + }, + { + "id": "local_v2_cold_replay", + "carrier": { + "outcome": "retained", + "current_policy_and_clock": "changed" + }, + "expected": { + "stored_disposition_and_availability_unchanged": true, + "new_retry_calculation_count": 0 + } + }, + { + "id": "local_v2_timestamp_overflow", + "carrier": { + "sample": "maximum bindable backend instant minus 500000 microseconds", + "lease_expiry": "maximum bindable backend instant", + "override": 1, + "maximum_interval_seconds": 1, + "original_deadline": null + }, + "expected": { + "recorded": false, + "reason": "invalid_activity_retry_delay", + "durable_mutation_count": 0, + "wrap_or_clamp": false + } + }, + { + "id": "local_v2_portable_not_duration_bound", + "carrier": { + "override": 9007199254740991, + "maximum_interval_seconds": 9007199254740991, + "original_deadline": null, + "condition": "selected duration or resulting backend timestamp not representable" + }, + "expected": { + "recorded": false, + "reason": "invalid_activity_retry_delay", + "durable_mutation_count": 0, + "wrap_or_clamp": false + } + }, + { + "id": "local_v2_legacy_zero_tail", + "carrier": { + "mode": "unprepared local contract 1", + "snapshot_version": 1, + "short_or_missing_backoff": "existing value" + }, + "expected": { + "legacy_interpretation_unchanged": true, + "local_v2_result": false + } + }, + { + "id": "local_v2_concurrent_identical_outcomes", + "carrier": { + "two_valid_fenced_outcomes": "same original attempt and normalized report", + "execution": "one canonical transaction serializes first" + }, + "expected": { + "committed_decision_count": 1, + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "winner_duplicate": false, + "loser_duplicate": true, + "loser_reuses_winner_sample_and_decision": true, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_concurrent_changed_override", + "carrier": { + "two_valid_fenced_outcomes": "same original attempt, distinct valid override integers", + "execution": "one canonical transaction serializes first" + }, + "expected": { + "committed_decision_count": 1, + "new_retry_task_count_at_most": 1, + "new_retry_event_count_at_most": 1, + "loser_recorded": false, + "loser_reason": "local_activity_outcome_mismatch", + "loser_mutation_count": 0, + "new_attempt_count": 0 + } + } + ] +} From b744ff976079113dceafa30e7341f0a05cb4d64f Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:07:31 +0000 Subject: [PATCH 05/15] Bind prepared timeout precedence and retained late replies --- docs/contracts/prepared-local-retry-v2.md | 5 +- .../prepared-local.v2.json | 61 +++++++++++++++++-- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index f13cdeef5..f28f6d6d4 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -37,7 +37,8 @@ exhausted outcomes select no delay. An already-due prepared engine timeout retains its engine provenance rather than borrowing the caller's application failure. An earlier valid failure whose selected availability reaches the total deadline remains an application failure with terminal retry-delay disposition. -General's reconciliation of that mode-bound projection is pending before freeze. +General accepted this mode-bound projection in +[the source decision](https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988). The object is proposed engine-owned output shared by outcome history and the matching receipt. It is not accepted worker input. Existing receipt identities, @@ -58,7 +59,7 @@ its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. Workflow owns the complete normalized snapshot and existing timeout/range binding. Server owns the companion validator, transaction, transport and storage -bindings. The exact output object, final normalized ranges, timeout projection +bindings. The exact output object, final normalized ranges and HTTP bindings must be reconciled before these bytes are frozen or SDKs adopt them. Native implementation, schema, installed/backend qualification and release remain separate work. diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 99b45723d..7eb659a72 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -19,10 +19,10 @@ "sha256": "c37fb132129e4557c7ffbd90404763924640262bb5ed10e764f090f69ae12bf5", "case_count": 37 }, - "prior_rows_mutated": false + "prior_rows_mutated": false, + "prepared_timeout_projection": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988" }, "adoption_gates": [ - "General reconciles the prepared-timeout projection before final companion freeze", "Core accepts the proposed closed retry_disposition fields and normalized snapshot/range binding", "Server binds the full HTTP envelope/refusal matrix to the installed future prepared implementation", "Native implements the prepared role and validator/transaction writer before advertisement", @@ -100,7 +100,14 @@ "backdated version-1 batch timestamps" ], "serialization": "UTC RFC3339 with exactly six fractional digits and Z; nullable original schedule-to-close deadline", - "prepared_timeout_projection": "preserve existing engine timeout precedence at an already-due deadline; a valid earlier application failure whose selected availability reaches the deadline remains an application failure with terminal retry-delay disposition" + "prepared_timeout_projection": "preserve existing engine timeout precedence at an already-due deadline; a valid earlier application failure whose selected availability reaches the deadline remains an application failure with terminal retry-delay disposition", + "precedence": [ + "strict report validation", + "matching retained outcome receipt", + "fresh original claim and temporal fences using one sample", + "actually due engine timeout", + "application failure retry selection" + ] }, "retry_disposition_proposal": { "boundary": "engine-owned object shared exactly by the stored outcome metadata and its prepared receipt; never accepted in a worker report", @@ -748,9 +755,9 @@ { "id": "local_v2_availability_equal_deadline", "carrier": { - "failure_time_seconds": 100, - "original_deadline_seconds": 105, - "override": 5, + "failure_time_seconds": 99, + "original_deadline_seconds": 100, + "override": 1, "maximum_interval_seconds": 20, "original_claim_live": true }, @@ -1029,6 +1036,48 @@ "loser_mutation_count": 0, "new_attempt_count": 0 } + }, + { + "id": "local_v2_late_duplicate_retains_pre_deadline_terminal", + "carrier": { + "first_valid_outcome": { + "failure_time_seconds": 99, + "original_deadline_seconds": 100, + "override": 1, + "committed": true + }, + "reply": "lost", + "identical_retry_sample_seconds": 100 + }, + "expected": { + "duplicate": true, + "retained_failure_time_seconds": 99, + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "original_application_failure_preserved": true, + "fresh_engine_timeout_calculation_count": 0, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_completed_at_deadline_engine_timeout", + "carrier": { + "engine_sample_seconds": 100, + "original_deadline_seconds": 100, + "submitted_outcome": "completed", + "result": "otherwise valid independently encoded Avro result", + "lease_and_run_deadlines": "later than sample" + }, + "expected": { + "event_type": "ActivityTimedOut", + "terminal_classification": "engine_timeout", + "submitted_completion_published": false, + "engine_timeout_provenance_preserved": true, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } } ] } From 02aee1bca25dad78569e80a038f05c97f175ff28 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:20:42 +0000 Subject: [PATCH 06/15] Model policy retries for engine local timeouts and explicit v2 bounds --- docs/contracts/prepared-local-retry-v2.md | 15 +- .../prepared-local.v2.json | 556 +++++++++++++++++- 2 files changed, 557 insertions(+), 14 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index f28f6d6d4..0ba722eef 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -33,9 +33,12 @@ The proposed `retry_disposition` object has closed state-dependent fields in the companion. Scheduled decisions retain candidate, cap and checked availability. Terminal cap/deadline decisions retain a diagnostic candidate but have no retry availability, task, start, following attempt or retry event. Non-retryable and -exhausted outcomes select no delay. An already-due prepared engine timeout -retains its engine provenance rather than borrowing the caller's application -failure. An earlier valid failure whose selected availability reaches the total +exhausted outcomes select no delay. An already-due overall schedule-to-close timeout retains its engine +provenance rather than borrowing the caller's application failure. An actual +start-to-close or heartbeat timeout before that total deadline may instead +retry under the engine policy when its attempt budget remains. The disposition +identifies the failure source, actual timeout kind and selected delay source; +a preempted application override cannot become the timeout retry delay. An earlier valid failure whose selected availability reaches the total deadline remains an application failure with terminal retry-delay disposition. General accepted this mode-bound projection in [the source decision](https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988). @@ -53,8 +56,10 @@ grammars, conjunctive mode gates, isolated authoritative identities, deadline edges, terminal precedence, reply loss, receipt mismatch, rollback, cold replay, checked backend timestamp arithmetic and unchanged version-1 behavior. They are requirements for a future real writer/adapter. Their presence or JSON consistency -is not an executing pass. The portable integer bound is distinct from an actual -backend duration or timestamp bound. Neither the old 100-report batch limit nor +is not an executing pass. The companion also binds Core's proposed ten-key stored snapshot and explicit +new-route attempt/list/type/timeout bounds. These proposed bounds are separate +from current version-1 facts. The portable integer bound is distinct from an +actual backend duration or timestamp bound. Neither the old 100-report batch limit nor its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. Workflow owns the complete normalized snapshot and existing timeout/range diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 7eb659a72..f5fde094c 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -20,7 +20,8 @@ "case_count": 37 }, "prior_rows_mutated": false, - "prepared_timeout_projection": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988" + "prepared_timeout_projection": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988", + "core_snapshot_range_proposal": "https://github.com/durable-workflow/workflow/pull/1038#issuecomment-6669717376" }, "adoption_gates": [ "Core accepts the proposed closed retry_disposition fields and normalized snapshot/range binding", @@ -100,7 +101,7 @@ "backdated version-1 batch timestamps" ], "serialization": "UTC RFC3339 with exactly six fractional digits and Z; nullable original schedule-to-close deadline", - "prepared_timeout_projection": "preserve existing engine timeout precedence at an already-due deadline; a valid earlier application failure whose selected availability reaches the deadline remains an application failure with terminal retry-delay disposition", + "prepared_timeout_projection": "preserve actual engine timeout precedence; start_to_close/heartbeat with remaining total time and attempt budget may retry under policy; actually due schedule_to_close terminates before application override selection", "precedence": [ "strict report validation", "matching retained outcome receipt", @@ -116,7 +117,8 @@ "snapshot_version": 2, "decision": "scheduled or terminal", "failure_time": "the stored first accepted engine sample", - "original_schedule_to_close_deadline_at": "stored UTC timestamp or null" + "original_schedule_to_close_deadline_at": "stored UTC timestamp or null", + "failure_source": "application or engine_timeout" }, "common_optional": { "requested_delay_seconds": "present iff the valid failed report supplied next_retry_delay_seconds; exact requested integer, including zero; diagnostic input does not prove application" @@ -124,7 +126,8 @@ "scheduled_required": { "candidate_delay_seconds": "selected nonnegative portable integer", "maximum_interval_seconds": "finite cap from retained snapshot 2", - "retry_available_at": "checked failure_time plus selected seconds, strictly before original deadline if set" + "retry_available_at": "checked failure_time plus selected seconds, strictly before original deadline if set", + "delay_source": "policy or failure_override; engine timeouts always policy" }, "scheduled_forbidden": [ "terminal_classification" @@ -144,7 +147,8 @@ ], "terminal_delay_required": [ "candidate_delay_seconds", - "maximum_interval_seconds" + "maximum_interval_seconds", + "delay_source" ], "terminal_without_selection": [ "non_retryable", @@ -158,7 +162,12 @@ "terminal_forbidden": [ "retry_available_at" ], - "unknown_members": "refused by a future consumer; never silently ignored" + "unknown_members": "refused by a future consumer; never silently ignored", + "conditional_common": { + "timeout_kind": "required iff failure_source is engine_timeout; one of start_to_close, schedule_to_close, heartbeat; forbidden otherwise" + }, + "delay_source_rule": "failure_override only for a still-admitted application failure with a supplied valid override; otherwise policy. A preempted application override stays requested diagnostic input only.", + "engine_timeout_terminal_classification": "reserved for overall schedule_to_close expiry; start_to_close/heartbeat before the total deadline may schedule the policy retry or terminate under their actual budget/cap/deadline branch" }, "engine_receipt_proposal": { "existing_required": [ @@ -214,7 +223,8 @@ "scheduled": "one ActivityRetryScheduled for the real failed attempt, with matching stored retry_disposition, retry_available_at and actual selected retry_backoff_seconds; no following attempt exists until independently prepared", "terminal": "one existing ActivityFailed or ActivityTimedOut outcome for the real attempt; matching retry_disposition; no retry task, following attempt/start or ActivityRetryScheduled", "failure_preservation": "cap/deadline retry termination preserves the admitted application failure/type; already-due engine timeout preserves engine timeout provenance instead", - "cold_replay": "consume the stored decision and availability; do not evaluate the current policy or clock" + "cold_replay": "consume the stored decision and availability; do not evaluate the current policy or clock", + "timeout_scheduled": "an actual pre-total-deadline start_to_close/heartbeat timeout may record ActivityRetryScheduled with retry_reason timeout, actual timeout_kind and policy-selected delay; no worker override is applied" }, "ranges": { "portable_integer_minimum": 0, @@ -224,7 +234,7 @@ "v2_inherits_these_batch_ceilings": false, "checked_selection": "a selected retry must also fit the backend duration arithmetic and resulting bindable UTC timestamp before writes; no wrap, clamp, float coercion or immediate fallback", "terminal_candidate": "a portable diagnostic integer is not an executed duration or retry availability", - "remaining_snapshot_binding": "Core must return exact policy/snapshot keys, finite max-attempt/list bounds and existing timeout field bounds; this proposal does not invent them from the batch limits" + "remaining_snapshot_binding": "Core proposed the exact ten-key snapshot, new-route bounds and closed policy above; final joint source reconciliation remains before freeze" }, "cases": [ { @@ -1078,6 +1088,534 @@ "new_retry_event_count": 0, "new_attempt_count": 0 } + }, + { + "id": "local_v2_start_to_close_timeout_policy_retry", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "start_to_close", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20, + "submitted_failed_override": 1 + }, + "expected": { + "decision": "scheduled", + "failure_source": "engine_timeout", + "timeout_kind": "start_to_close", + "delay_source": "policy", + "candidate_delay_seconds": 5, + "retry_available_at_seconds": 95, + "submitted_override_applied": false, + "event_type": "ActivityRetryScheduled", + "retry_reason": "timeout", + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_start_to_close_timeout_exhausted", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "start_to_close", + "attempt_number": 3, + "max_attempts": 3 + }, + "expected": { + "decision": "terminal", + "failure_source": "engine_timeout", + "timeout_kind": "start_to_close", + "terminal_classification": "attempts_exhausted", + "candidate_delay_member": "absent", + "event_type": "ActivityTimedOut", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_start_to_close_timeout_policy_reaches_deadline", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "start_to_close", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 10, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "failure_source": "engine_timeout", + "timeout_kind": "start_to_close", + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "delay_source": "policy", + "candidate_delay_seconds": 10, + "event_type": "ActivityTimedOut", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_heartbeat_timeout_policy_retry", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "heartbeat", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20, + "submitted_failed_override": 1 + }, + "expected": { + "decision": "scheduled", + "failure_source": "engine_timeout", + "timeout_kind": "heartbeat", + "delay_source": "policy", + "candidate_delay_seconds": 5, + "retry_available_at_seconds": 95, + "submitted_override_applied": false, + "event_type": "ActivityRetryScheduled", + "retry_reason": "timeout", + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_heartbeat_timeout_exhausted", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "heartbeat", + "attempt_number": 3, + "max_attempts": 3 + }, + "expected": { + "decision": "terminal", + "failure_source": "engine_timeout", + "timeout_kind": "heartbeat", + "terminal_classification": "attempts_exhausted", + "candidate_delay_member": "absent", + "event_type": "ActivityTimedOut", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_heartbeat_timeout_policy_reaches_deadline", + "carrier": { + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actual_timeout_kind": "heartbeat", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 10, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "failure_source": "engine_timeout", + "timeout_kind": "heartbeat", + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "delay_source": "policy", + "candidate_delay_seconds": 10, + "event_type": "ActivityTimedOut", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } + }, + { + "id": "local_v2_timed_out_report_not_actually_due", + "carrier": { + "report": { + "outcome": "timed_out" + }, + "all_actual_timeout_deadlines": "later than engine sample" + }, + "expected": { + "recorded": false, + "reason": "local_activity_timeout_not_due", + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_range_max_attempts_0", + "carrier": { + "isolated_policy_or_snapshot_boundary": "max_attempts", + "value": 0, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_max_attempts_1", + "carrier": { + "isolated_policy_or_snapshot_boundary": "max_attempts", + "value": 1, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_max_attempts_1024", + "carrier": { + "isolated_policy_or_snapshot_boundary": "max_attempts", + "value": 1024, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_max_attempts_1025", + "carrier": { + "isolated_policy_or_snapshot_boundary": "max_attempts", + "value": 1025, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_backoff_list_length_1024", + "carrier": { + "isolated_policy_or_snapshot_boundary": "backoff_list_length", + "value": 1024, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_backoff_list_length_1025", + "carrier": { + "isolated_policy_or_snapshot_boundary": "backoff_list_length", + "value": 1025, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_error_type_count_128", + "carrier": { + "isolated_policy_or_snapshot_boundary": "error_type_count", + "value": 128, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_error_type_count_129", + "carrier": { + "isolated_policy_or_snapshot_boundary": "error_type_count", + "value": 129, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_error_type_utf8_bytes_255", + "carrier": { + "isolated_policy_or_snapshot_boundary": "error_type_utf8_bytes", + "value": 255, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_error_type_utf8_bytes_256", + "carrier": { + "isolated_policy_or_snapshot_boundary": "error_type_utf8_bytes", + "value": 256, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_timeout_seconds_0", + "carrier": { + "isolated_policy_or_snapshot_boundary": "timeout_seconds", + "value": 0, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_timeout_seconds_1", + "carrier": { + "isolated_policy_or_snapshot_boundary": "timeout_seconds", + "value": 1, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_timeout_seconds_2147483647", + "carrier": { + "isolated_policy_or_snapshot_boundary": "timeout_seconds", + "value": 2147483647, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_timeout_seconds_2147483648", + "carrier": { + "isolated_policy_or_snapshot_boundary": "timeout_seconds", + "value": 2147483648, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_delay_seconds_9007199254740991", + "carrier": { + "isolated_policy_or_snapshot_boundary": "delay_seconds", + "value": 9007199254740991, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": true, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_range_delay_seconds_9007199254740992", + "carrier": { + "isolated_policy_or_snapshot_boundary": "delay_seconds", + "value": 9007199254740992, + "other_shapes": "valid", + "representable_deadline": "independent subsequent guard if applicable" + }, + "expected": { + "shape_range_accepted": false, + "durable_mutation_count_if_refused": 0, + "runtime_result": null + } + }, + { + "id": "local_v2_snapshot_missing_required_key", + "carrier": { + "isolated_closed_shape_mutation": "missing_required_key" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_unknown_key", + "carrier": { + "isolated_closed_shape_mutation": "unknown_key" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_Boolean_numeric", + "carrier": { + "isolated_closed_shape_mutation": "Boolean_numeric" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_null_attempt_budget", + "carrier": { + "isolated_closed_shape_mutation": "null_attempt_budget" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_non_null_schedule_to_start_timeout", + "carrier": { + "isolated_closed_shape_mutation": "non_null_schedule_to_start_timeout" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_wire_generator_descriptor", + "carrier": { + "isolated_closed_shape_mutation": "wire_generator_descriptor" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_worker_snapshot_version", + "carrier": { + "isolated_closed_shape_mutation": "worker_snapshot_version" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_snapshot_oversized_generated_budget_before_expansion", + "carrier": { + "isolated_closed_shape_mutation": "oversized_generated_budget_before_expansion" + }, + "expected": { + "snapshot_or_policy": "refused", + "allocation_after_refusal": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_terminal_huge_candidate_is_not_duration", + "carrier": { + "original_claim_live": true, + "failure_source": "application", + "requested_delay_seconds": 9007199254740991, + "maximum_interval_seconds": 20 + }, + "expected": { + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_policy_cap", + "candidate_delay_seconds": 9007199254740991, + "retry_available_at_member": "absent", + "duration_conversion_for_terminal_candidate": false, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0 + } } - ] + ], + "prepared_policy_proposal": { + "required": { + "contract_version": 2, + "mode": [ + "explicit", + "generated" + ], + "backoff_seconds": "ordered list of 0..1024 actual integers in 0..9007199254740991" + }, + "optional": { + "max_attempts": "actual integer 1..1024, default 1", + "maximum_interval_seconds": "actual integer 0..9007199254740991; canonical mode-specific default", + "non_retryable_error_types": "0..128 valid UTF-8 nonblank strings, each <=255 bytes before existing trim/first-occurrence deduplication; default []" + }, + "unknown_members": "refuse, including snapshot_version and generator descriptors", + "generator_boundary": "bounded authoring expansion precedes the wire; do not infer or reconstruct a generator from a flattened worker list", + "generated_schedule_positions_maximum": 1023 + }, + "stored_snapshot_proposal": { + "required": { + "snapshot_version": 2, + "mode": [ + "explicit", + "generated" + ], + "max_attempts": "actual integer 1..1024", + "backoff_seconds": "ordered list of 0..1024 actual integers in 0..9007199254740991", + "maximum_interval_seconds": "actual integer 0..9007199254740991", + "start_to_close_timeout": "null or actual integer 1..2147483647", + "schedule_to_start_timeout": null, + "schedule_to_close_timeout": "null or actual integer 1..2147483647", + "heartbeat_timeout": "null or actual integer 1..2147483647", + "non_retryable_error_types": "ordered list of 0..128 normalized error types, each valid UTF-8 nonblank <=255 original bytes" + }, + "optional": [], + "unknown_members": "refuse", + "unlimited_null_or_sentinel_attempt_budget": "not part of version 2; preserve existing legacy path", + "preparation_deadlines": "checked backend timestamp arithmetic before the original execution/start authority is published" + } } From 87baf1c9c46dd7a380d45a698b72778e4be45b6b Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:25:12 +0000 Subject: [PATCH 07/15] Close generated wire tails and no-selection disposition fields --- .../prepared-local.v2.json | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index f5fde094c..148fdff3a 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -157,7 +157,8 @@ ], "terminal_without_selection_forbidden": [ "candidate_delay_seconds", - "maximum_interval_seconds" + "maximum_interval_seconds", + "delay_source" ], "terminal_forbidden": [ "retry_available_at" @@ -1577,6 +1578,53 @@ "new_retry_event_count": 0, "new_attempt_count": 0 } + }, + { + "id": "local_v2_flattened_generated_wire_tail_preserved", + "carrier": { + "wire_policy_mode": "generated", + "max_attempts": 1024, + "backoff_list_length": 1024, + "all_entries": "valid and no larger than supplied finite cap" + }, + "expected": { + "wire_shape": "accepted", + "stored_list_length": 1024, + "unused_supplied_tail_preserved": true, + "invented_exact_length_rule": false, + "runtime_result": null + } + }, + { + "id": "local_v2_authoring_generated_expansion_positions", + "carrier": { + "boundary": "authoring expansion before flattened worker wire", + "max_attempts": 1024 + }, + "expected": { + "maximum_generated_retry_positions": 1023, + "first_attempt_included_in_budget": true, + "before_work_bound_checked": true, + "flattened_wire_exact_length_inferred": false, + "runtime_result": null + } + }, + { + "id": "local_v2_terminal_without_selection_delay_source_forbidden", + "carrier": { + "otherwise_valid_disposition": { + "decision": "terminal", + "terminal_classification": "attempts_exhausted" + }, + "extra_member": { + "delay_source": "policy" + } + }, + "expected": { + "disposition_shape": "refused", + "selected_delay_claim": false, + "runtime_result": null + } } ], "prepared_policy_proposal": { @@ -1595,7 +1643,8 @@ }, "unknown_members": "refuse, including snapshot_version and generator descriptors", "generator_boundary": "bounded authoring expansion precedes the wire; do not infer or reconstruct a generator from a flattened worker list", - "generated_schedule_positions_maximum": 1023 + "generated_schedule_positions_maximum": 1023, + "flattened_generated_wire_tail": "the shared <=1024 list bound is intentional; retain every admitted supplied entry, including unused tails; no inferred max_attempts-1 exact-length wire restriction" }, "stored_snapshot_proposal": { "required": { From aea7b43b136b44202ecb5100f395ed271b1ab7a9 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:00:52 +0000 Subject: [PATCH 08/15] Propose closed local retry HTTP replies and effect-free preflight --- docs/contracts/prepared-local-retry-v2.md | 40 +- .../prepared-local.v2.json | 738 +++++++++++++++++- 2 files changed, 763 insertions(+), 15 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index 0ba722eef..b10eab0c0 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -23,7 +23,7 @@ wall-clock failure time, deadline, policy snapshot or engine timeout category. The engine uses one UTC sample under the canonical locks for fresh temporal fences and retry selection. The first transaction that commits retains that sample and decision. Rollback publishes neither. A matching outcome receipt -replays its stored decision before fresh retry calculation. The original total +replays its stored failed/timeout retry decision before fresh calculation. The original total deadline comes from the prepared execution and correlated start authority, never a duplicate response's fresh `server_time` or a batch's recording clock. The SDK must receive or recover that engine decision before sleeping or invoking @@ -33,12 +33,14 @@ The proposed `retry_disposition` object has closed state-dependent fields in the companion. Scheduled decisions retain candidate, cap and checked availability. Terminal cap/deadline decisions retain a diagnostic candidate but have no retry availability, task, start, following attempt or retry event. Non-retryable and -exhausted outcomes select no delay. An already-due overall schedule-to-close timeout retains its engine +exhausted outcomes select no delay. An already-due overall schedule-to-close +timeout retains its engine provenance rather than borrowing the caller's application failure. An actual start-to-close or heartbeat timeout before that total deadline may instead retry under the engine policy when its attempt budget remains. The disposition identifies the failure source, actual timeout kind and selected delay source; -a preempted application override cannot become the timeout retry delay. An earlier valid failure whose selected availability reaches the total +a preempted application override cannot become the timeout retry delay. An +earlier valid failure whose selected availability reaches the total deadline remains an application failure with terminal retry-delay disposition. General accepted this mode-bound projection in [the source decision](https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988). @@ -46,20 +48,40 @@ General accepted this mode-bound projection in The object is proposed engine-owned output shared by outcome history and the matching receipt. It is not accepted worker input. Existing receipt identities, claim-release behavior and created-task identity remain authoritative. The -companion closes this engine object, not the whole HTTP response. The controller's -history cursor, protocol envelope and exact refusal/status matrix still need a -separate source binding before adoption. No new top-level HTTP reason aliases -are selected here. +companion now also proposes complete HTTP response families, the fixed +zero-history cursor, pinned protocol envelope and finite status/refusal cases. +Those templates still need joint source review and an actual future adapter. +They retain the distinct controller-missing, controller-stale and structured +engine refusals rather than accepting arbitrary `recorded:false` shapes. +No new top-level reason alias is selected. + +The future handler must run a pure report shape/type/override preflight before +its task, attempt, original-claim or registration lookup. Current Server checks +only that the report is an array before those lookups, so Core's later strict +normalizer alone does not establish this ordering. Recheck admitted shape and +fences under the authoritative transaction. The pure preflight cannot read +storage, fetch payloads, invoke a callback or sample the failure clock. +Core's entire current normalizer cannot be lifted into this phase: its +completed branch resolves payload envelopes. + +The retained fetch-free retry-decision recovery applies to failed and timed-out +reports with pure normalized fingerprints. Completed external-reference +normalization retains its existing separately scoped payload contract. This +feature introduces no fetch-free completed replay guarantee or new retained +external-artifact identity. Missing resolution or an uncertain response never +grants permission to repeat a callback. The data's named source controls cover strict override types, separate report grammars, conjunctive mode gates, isolated authoritative identities, deadline edges, terminal precedence, reply loss, receipt mismatch, rollback, cold replay, checked backend timestamp arithmetic and unchanged version-1 behavior. They are requirements for a future real writer/adapter. Their presence or JSON consistency -is not an executing pass. The companion also binds Core's proposed ten-key stored snapshot and explicit +is not an executing pass. The companion also binds Core's proposed ten-key +stored snapshot and explicit new-route attempt/list/type/timeout bounds. These proposed bounds are separate from current version-1 facts. The portable integer bound is distinct from an -actual backend duration or timestamp bound. Neither the old 100-report batch limit nor +actual backend duration or timestamp bound. Neither the old 100-report batch +limit nor its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. Workflow owns the complete normalized snapshot and existing timeout/range diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 148fdff3a..83b5ea3c1 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -8,10 +8,13 @@ "native_role_inventory": "https://github.com/durable-workflow/server/pull/1034#issuecomment-6668962888", "native_reference": "d8eb3bc1a5368752be823784521a9a09ec44de71", "canonical": { - "commit": "0aa1536e6198a6667ddedf62b48215b6e9e8a6f8", + "commit": "8db66b9f0ec683386c4b4cf698c5e38e46d44536", "path": "resources/protocol/activity-retry-policy-v2-vectors.json", "sha256": "8d576f1d50bcbfb123cb09682a4fbac53cb018a3cd369021c38d69866849781c", - "case_count": 54 + "case_count": 54, + "original_reviewed_commit": "0aa1536e6198a6667ddedf62b48215b6e9e8a6f8", + "blob": "4e1d6d9c4fc619b9d0ba0430d5aa06f250c4f61e", + "identity_scope": "exact accepted-main source bytes only; no v2 producer/runtime or full-main-QA claim" }, "remote_companion": { "commit": "9bc440bf222f9cfc9e4401a3ce3b67246bf558de", @@ -25,7 +28,7 @@ }, "adoption_gates": [ "Core accepts the proposed closed retry_disposition fields and normalized snapshot/range binding", - "Server binds the full HTTP envelope/refusal matrix to the installed future prepared implementation", + "Core/Server reconcile the proposed HTTP families, pure preflight placement and separate completed payload scope before final source freeze", "Native implements the prepared role and validator/transaction writer before advertisement", "Actual native/database/SDK/installed qualification and release remain separate" ], @@ -92,7 +95,7 @@ "clock": { "sample": "one engine UTC sample after strict request validation and canonical authority locks; the same sample drives fresh temporal fences and the retry decision", "publication": "retained only with the first transaction that commits; rollback publishes neither a sample nor decision", - "duplicate": "return the matching committed per-attempt receipt and decision; never resample or rerun policy for that receipt", + "duplicate": "for matching committed failed/timed_out retry decisions, return the original sample/decision without fresh selection; completed payload replay retains its existing separate resolution contract", "original_deadline": "stored prepared execution schedule_to_close_deadline_at, checked against correlated original preparation/start authority", "not_anchors": [ "duplicate prepare response server_time", @@ -217,14 +220,14 @@ "created_task_ids": [], "retry_disposition": "absent" }, - "http_boundary": "this closes the engine receipt object only; controller history_refresh_page_token, protocol envelope and refusal HTTP status/body require their own exact source binding before adoption" + "http_boundary": "the proposed full reply envelopes and distinct refusal families are declared in http_proposal; they still require actual future handler/adapter implementation and qualification" }, "history": { "identity": "same run/execution/real attempt/worker attempt/original task epoch as the prepared outcome receipt", "scheduled": "one ActivityRetryScheduled for the real failed attempt, with matching stored retry_disposition, retry_available_at and actual selected retry_backoff_seconds; no following attempt exists until independently prepared", "terminal": "one existing ActivityFailed or ActivityTimedOut outcome for the real attempt; matching retry_disposition; no retry task, following attempt/start or ActivityRetryScheduled", "failure_preservation": "cap/deadline retry termination preserves the admitted application failure/type; already-due engine timeout preserves engine timeout provenance instead", - "cold_replay": "consume the stored decision and availability; do not evaluate the current policy or clock", + "cold_replay": "consume stored failed/timeout retry disposition and availability; never evaluate current policy/clock for that decision; no fetch-free completed replay claim", "timeout_scheduled": "an actual pre-total-deadline start_to_close/heartbeat timeout may record ActivityRetryScheduled with retry_reason timeout, actual timeout_kind and policy-selected delay; no worker override is applied" }, "ranges": { @@ -1625,6 +1628,496 @@ "selected_delay_claim": false, "runtime_result": null } + }, + { + "id": "local_v2_http_unsupported_server_literal_capability", + "carrier": { + "fresh_v2_mode_gate": "server_literal_capability", + "other_gates": "valid installed prepared v2 baseline", + "report": "otherwise valid failed report" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "activity_retry_delay_override_v1", + "unavailable": [ + "activity_retry_delay_override_v1" + ], + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_http_unsupported_original_local_contract_version", + "carrier": { + "fresh_v2_mode_gate": "original_local_contract_version", + "other_gates": "valid installed prepared v2 baseline", + "report": "otherwise valid failed report" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "activity_retry_delay_override_v1", + "unavailable": [ + "local_activity_contract_version" + ], + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_http_unsupported_retained_snapshot_version", + "carrier": { + "fresh_v2_mode_gate": "retained_snapshot_version", + "other_gates": "valid installed prepared v2 baseline", + "report": "otherwise valid failed report" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "activity_retry_delay_override_v1", + "unavailable": [ + "activity_retry_snapshot_version" + ], + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_http_unsupported_original_claim_capability", + "carrier": { + "fresh_v2_mode_gate": "original_claim_capability", + "other_gates": "valid installed prepared v2 baseline", + "report": "otherwise valid failed report" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "activity_retry_delay_override_v1", + "unavailable": [ + "original_claim_activity_retry_delay_override_v1" + ], + "durable_mutation_count": 0, + "next_callback_count": 0 + } + }, + { + "id": "local_v2_http_current_native_role_absent", + "carrier": { + "source": "current native reference", + "protocol": "1.20", + "report": "otherwise valid" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "prepared_local_activities", + "unavailable": [ + "installed_runtime_prepared_local_activity" + ], + "local_v2_path_qualified": false + } + }, + { + "id": "local_v2_http_malformed_override_owned_live", + "carrier": { + "namespace_auth": "valid", + "isolated_authority_state": "owned_live", + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": null + } + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "durable_mutation_count": 0, + "retained_receipt_bypasses_strict_shape": false + } + }, + { + "id": "local_v2_http_malformed_override_owned_stale", + "carrier": { + "namespace_auth": "valid", + "isolated_authority_state": "owned_stale", + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": null + } + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "durable_mutation_count": 0, + "retained_receipt_bypasses_strict_shape": false + } + }, + { + "id": "local_v2_http_malformed_override_foreign_missing", + "carrier": { + "namespace_auth": "valid", + "isolated_authority_state": "foreign_missing", + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": null + } + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "durable_mutation_count": 0, + "retained_receipt_bypasses_strict_shape": false + } + }, + { + "id": "local_v2_http_malformed_override_matching_old_receipt", + "carrier": { + "namespace_auth": "valid", + "isolated_authority_state": "matching_old_receipt", + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": null + } + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "durable_mutation_count": 0, + "retained_receipt_bypasses_strict_shape": false + } + }, + { + "id": "local_v2_http_controller_missing", + "carrier": { + "otherwise_valid": true, + "actual_stage_and_condition": "controller_missing" + }, + "expected": { + "http_status": 404, + "response_family": "controller_attempt_missing", + "reason": "activity_attempt_not_found", + "durable_mutation_count": 0, + "generic_recorded_false_shape_accepted": false + } + }, + { + "id": "local_v2_http_controller_stale", + "carrier": { + "otherwise_valid": true, + "actual_stage_and_condition": "controller_stale" + }, + "expected": { + "http_status": 409, + "response_family": "controller_registration_stale", + "reason": "stale_worker_registration", + "durable_mutation_count": 0, + "generic_recorded_false_shape_accepted": false + } + }, + { + "id": "local_v2_http_engine_stale_attempt", + "carrier": { + "otherwise_valid": true, + "actual_stage_and_condition": "engine_stale_attempt" + }, + "expected": { + "http_status": 409, + "response_family": "engine_refusal", + "reason": "stale_activity_attempt", + "durable_mutation_count": 0, + "generic_recorded_false_shape_accepted": false + } + }, + { + "id": "local_v2_http_changed_retained_reply", + "carrier": { + "otherwise_valid": true, + "actual_stage_and_condition": "changed_retained_reply" + }, + "expected": { + "http_status": 409, + "response_family": "engine_refusal", + "reason": "local_activity_outcome_mismatch", + "durable_mutation_count": 0, + "generic_recorded_false_shape_accepted": false + } + }, + { + "id": "local_v2_http_first_scheduled", + "carrier": { + "actual_case_decision": "first_scheduled", + "otherwise_valid": true + }, + "expected": { + "http_status": 200, + "response_family": "recorded_retry_decision", + "event_type": "ActivityRetryScheduled", + "duplicate": false, + "outer_reason": null, + "exact_stored_disposition_required": true, + "closed_identity_binding_required": true + } + }, + { + "id": "local_v2_http_first_terminal", + "carrier": { + "actual_case_decision": "first_terminal", + "otherwise_valid": true + }, + "expected": { + "http_status": 200, + "response_family": "recorded_retry_decision", + "event_type": "ActivityFailed", + "duplicate": false, + "outer_reason": null, + "exact_stored_disposition_required": true, + "closed_identity_binding_required": true + } + }, + { + "id": "local_v2_http_engine_timeout_terminal", + "carrier": { + "actual_case_decision": "engine_timeout_terminal", + "otherwise_valid": true + }, + "expected": { + "http_status": 200, + "response_family": "recorded_retry_decision", + "event_type": "ActivityTimedOut", + "duplicate": false, + "outer_reason": null, + "exact_stored_disposition_required": true, + "closed_identity_binding_required": true + } + }, + { + "id": "local_v2_http_retained_late_terminal", + "carrier": { + "actual_case_decision": "retained_late_terminal", + "otherwise_valid": true + }, + "expected": { + "http_status": 200, + "response_family": "recorded_retry_decision", + "event_type": "ActivityFailed", + "duplicate": true, + "outer_reason": null, + "exact_stored_disposition_required": true, + "closed_identity_binding_required": true + } + }, + { + "id": "local_v2_http_response_extra_key", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "extra top-level member" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_wrong_reason_type", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "reason false instead of null" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_missing_protocol", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "missing protocol_version" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_mixed_stale_engine", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "controller stale body mixed with engine refusal fields" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_duplicate_identical", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "identical duplicate raw JSON member" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_duplicate_conflicting", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "conflicting duplicate raw JSON member" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_invalid_utf8", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "invalid original UTF-8 bytes" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_http_response_trailing_payload", + "carrier": { + "otherwise_exact_positive_response": true, + "isolated_mutation": "exact positive JSON followed by nonwhitespace or another JSON value" + }, + "expected": { + "response_binding": "refused before authoritative decision acceptance", + "new_callback_permission": false, + "runtime_result": null + } + }, + { + "id": "local_v2_shape_preflight_completed_reference_no_fetch", + "carrier": { + "report": "completed with structurally valid external envelope", + "scoped_claim_authority": "not yet read" + }, + "expected": { + "shape_phase_only": true, + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "callback_count": 0, + "failure_clock_sample_count": 0, + "payload_authority_or_bytes_qualified": false + } + }, + { + "id": "local_v2_completed_reference_replay_not_fetch_free", + "carrier": { + "completed_report": "external reference", + "initial_receipt": "may exist", + "fresh_resolution": "unavailable" + }, + "expected": { + "retry_decision_recovery_guarantee_applies": false, + "completed_payload_contract": "existing separate gate", + "repeat_callback_permission": false + } + }, + { + "id": "local_v2_http_backend_failure_not_semantic_pass", + "carrier": { + "otherwise_valid": true, + "response": "generic backend/transport failure" + }, + "expected": { + "semantic_negative_pass": false, + "decision": "unknown", + "repeat_callback_permission": false + } + }, + { + "id": "local_v2_http_raw_nan", + "carrier": { + "otherwise_valid_request": true, + "override_json_lexeme": "NaN" + }, + "expected": { + "http_status": 400, + "response_family": "native_malformed_json", + "typed_override_verdict": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_http_raw_infinity", + "carrier": { + "otherwise_valid_request": true, + "override_json_lexeme": "Infinity" + }, + "expected": { + "http_status": 400, + "response_family": "native_malformed_json", + "typed_override_verdict": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_http_raw_overflowing_exponent", + "carrier": { + "otherwise_valid_request": true, + "override_json_lexeme": "1e999" + }, + "expected": { + "http_status": 400, + "response_family": "native_malformed_json", + "typed_override_verdict": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_http_raw_invalid_utf8", + "carrier": { + "raw_request_hex": "7b227265706f7274223a22ff227d" + }, + "expected": { + "http_status": 400, + "response_family": "native_malformed_json", + "typed_override_verdict": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_http_attempt_removed_before_transaction", + "carrier": { + "earlier_controller_lookup": "original rows valid", + "serialized_legal_cleanup": "original attempt absent at canonical engine lookup, with real backend constraints respected" + }, + "expected": { + "http_status": 404, + "response_family": "engine_attempt_missing_after_controller_lookup", + "reason": "activity_attempt_not_found", + "controller_missing_shape_accepted": false, + "setup_or_backend_error_counts_as_pass": false + } } ], "prepared_policy_proposal": { @@ -1666,5 +2159,238 @@ "unknown_members": "refuse", "unlimited_null_or_sentinel_attempt_budget": "not part of version 2; preserve existing legacy path", "preparation_deadlines": "checked backend timestamp arithmetic before the original execution/start authority is published" + }, + "http_proposal": { + "status": "closed source templates proposed for the future v2 prepared outcome path; not implemented HTTP behavior", + "source": { + "server_commit": "cf069f6396f29e8f90563a6d75108779e30c3a2a", + "controller": { + "path": "app/Http/Controllers/Api/PreparedLocalActivityController.php", + "blob": "e6ef5eae06a6734e4c016ed8ad096315dc5b1f61" + }, + "protocol": { + "path": "app/Support/WorkerProtocol.php", + "blob": "1c40943935662888340149b1d7432aa5a0da169f" + }, + "native_missing_role": { + "path": "rust/src/runtime/prepared_local.rs", + "blob": "5b4f04eecc9f2beef202936c3c331a0f6a96f780" + }, + "native_json": { + "path": "rust/src/runtime/json_body.rs", + "blob": "50664ec66ce1cee1c8ecaf1d5a2d52a2f21a772c" + } + }, + "baseline": "valid independent auth/namespace/protocol and original prepared graph; all unrelated gates must be live before an isolated semantic control can qualify", + "request_scope": "existing controller outer claim/report validation and normalization remain separate; this proposal closes the nested v2 report and response families, not a new global protocol", + "pure_preflight": { + "placement": "after existing auth/protocol/runtime-role admission, before namespace task/attempt/claim/registration lookup", + "checks": [ + "closed report keys", + "actual field types", + "failed-only strict optional override", + "inline or reference envelope structural form only" + ], + "forbidden": [ + "storage read or write", + "payload or reference resolution", + "external fetch", + "callback", + "failure-clock sample" + ], + "not_reusable_as_is": "Core normalizePortableReport resolves completed payload envelopes before returning; only its pure shape/type interpretation may be shared" + }, + "transaction": "recheck admitted shape, immutable original mode/identity, matching retained failed/timeout decision and fresh fences under the existing canonical locks; no controller-only validation grants writer authority", + "bindings": { + "$protocol_version": "independently pinned source-under-test server worker protocol; reference is 1.20, never copied from the response", + "$server_capabilities": "complete independently pinned source-under-test manifest with exact types, never learned from the response", + "$path_attempt_id": "submitted path attempt ID, not a foreign looked-up identity", + "$worker_id": "validated submitted lease_owner string", + "$reason": "only the selected template/case fixed reason below", + "$event_id": "the unique matching canonical after-snapshot event, independently read with exact run/attempt/sequence binding", + "$event_type": "the event kind required by the case, not the response", + "$workflow_run_id": "original prepared run", + "$workflow_task_id": "original issued task", + "$workflow_task_attempt": "original issued task epoch", + "$activity_execution_id": "original prepared execution", + "$activity_attempt_id": "original prepared attempt", + "$worker_attempt_id": "original supplied worker attempt identity", + "$recorded_at": "the matching stored event timestamp", + "$created_task_ids": "exact independently observed new retry-task ID list, one for scheduled and empty for terminal/completed", + "$retry_disposition": "the complete engine object declared above, deep-equal to independently observed retained metadata", + "$required_capability": "prepared_local_activities for existing role/original-prepared-claim refusal; activity_retry_delay_override_v1 only for proposed fresh v2 mode-gate refusal", + "$unavailable": "the exact ordered array explicitly chosen by the case; no wildcard members", + "$duplicate_boolean": "actual Boolean required by the case, false for first commit and true for identical retained reply", + "$case_claim_released_boolean": "actual Boolean, true only for ActivityRetryScheduled, false for terminal/completed" + }, + "matching": "resolve declared bindings only; compare full exact JSON keys/types/values and the protocol response header; response decoding must consume strict UTF-8 original bytes fully and refuse duplicate member names before schema matching", + "envelope": { + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "cursor": "MA==", + "current_native": "prepared role remains false and returns the existing missing-role refusal; it cannot qualify future positive decisions or paired semantic negatives", + "completed_payload_scope": "preserve the existing completed/external-payload normalization and authority contract as a separate gate; no fetch-free completed replay guarantee is introduced", + "failed_timeout_recovery": "pure normalized failed/timed_out fingerprints allow matching committed retry decisions before fresh effects; do not extend that guarantee to completed external references", + "families": { + "recorded_retry_decision": { + "status": 200, + "body": { + "recorded": true, + "duplicate": "$duplicate_boolean", + "reason": null, + "event_id": "$event_id", + "event_type": "$event_type", + "workflow_run_id": "$workflow_run_id", + "workflow_task_id": "$workflow_task_id", + "workflow_task_attempt": "$workflow_task_attempt", + "activity_execution_id": "$activity_execution_id", + "activity_attempt_id": "$activity_attempt_id", + "worker_attempt_id": "$worker_attempt_id", + "recorded_at": "$recorded_at", + "claim_released": "$case_claim_released_boolean", + "created_task_ids": "$created_task_ids", + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities", + "retry_disposition": "$retry_disposition" + } + }, + "recorded_completed_existing_contract": { + "status": 200, + "body": { + "recorded": true, + "duplicate": "$duplicate_boolean", + "reason": null, + "event_id": "$event_id", + "event_type": "$event_type", + "workflow_run_id": "$workflow_run_id", + "workflow_task_id": "$workflow_task_id", + "workflow_task_attempt": "$workflow_task_attempt", + "activity_execution_id": "$activity_execution_id", + "activity_attempt_id": "$activity_attempt_id", + "worker_attempt_id": "$worker_attempt_id", + "recorded_at": "$recorded_at", + "claim_released": "$case_claim_released_boolean", + "created_task_ids": "$created_task_ids", + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "retry_disposition_member": "absent; no newly selected failure/timeout decision" + }, + "engine_refusal": { + "status": 409, + "body": { + "recorded": false, + "duplicate": false, + "reason": "$reason", + "activity_attempt_id": "$path_attempt_id", + "claim_released": false, + "created_task_ids": [], + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "reasons": [ + "invalid_local_activity_outcome", + "local_activity_outcome_requires_protocol_1_20", + "workflow_claim_not_found", + "local_activity_preparation_mismatch", + "local_activity_outcome_mismatch", + "stale_activity_attempt", + "run_closed", + "run_deadline_expired", + "workflow_claim_expired", + "workflow_claim_mismatch", + "local_activity_timeout_not_due", + "local_activity_cleanup_authority_expired" + ] + }, + "engine_attempt_missing_after_controller_lookup": { + "status": 404, + "body": { + "recorded": false, + "duplicate": false, + "reason": "activity_attempt_not_found", + "activity_attempt_id": "$path_attempt_id", + "claim_released": false, + "created_task_ids": [], + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + } + }, + "controller_attempt_missing": { + "status": 404, + "body": { + "reason": "activity_attempt_not_found", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + } + }, + "controller_registration_stale": { + "status": 409, + "body": { + "recorded": false, + "reason": "stale_worker_registration", + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + } + }, + "prepared_capability_refusal": { + "status": 409, + "body": { + "reason": "prepared_local_activity_not_supported", + "required_capability": "$required_capability", + "minimum_protocol_version": "1.20", + "worker_id": "$worker_id", + "unavailable": "$unavailable", + "remediation": "Use a prepared-local backend and a claim issued to a capable protocol 1.20 worker.", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "recorded_and_cursor_members": "absent, not false or null" + }, + "strict_override_refusal": { + "status": 422, + "body": { + "reason": "invalid_activity_retry_delay", + "message": "invalid_activity_retry_delay", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "recorded_next_task_cursor_members": "absent" + }, + "native_malformed_json": { + "status": 400, + "body": { + "message": "Request bodies must contain valid JSON.", + "reason": "malformed_json", + "json_error": "Syntax error", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities" + }, + "typed_override_verdict": false + } + }, + "mode_gate_unavailable_proposal": { + "server_literal_capability": "activity_retry_delay_override_v1", + "original_local_contract_version": "local_activity_contract_version", + "retained_snapshot_version": "activity_retry_snapshot_version", + "original_claim_capability": "original_claim_activity_retry_delay_override_v1" + }, + "unknown_or_transport_result": "missing, partial, malformed, contradictory, generic backend failure or unknown family is unqualified/uncertain; never permission to repeat a callback", + "response_headers": { + "X-Durable-Workflow-Protocol-Version": "$protocol_version", + "Content-Type": "application/json media type; unrelated standard headers are not an equality contract" + } + }, + "canonical_projection": { + "override": "prepared failed report.next_retry_delay_seconds maps to canonical application failure.next_retry_delay_seconds only", + "failure_type": "the normalized admitted application exception_type; engine timeouts retain their own source/type", + "time": "canonical whole-second examples map into the single stored authoritative UTC microsecond sample and immutable deadline; do not truncate subsecond comparisons", + "preempted_row": "zero_at_original_deadline_is_terminal remains an abstract selector row, preempted by actual overall engine timeout in the prepared HTTP path" } } From f3733db20ed2616a2b9aa4c64df5712899d686ee Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:02:19 +0000 Subject: [PATCH 09/15] Scope HTTP preflight counters after existing authentication --- .../contracts/activity-retry-policy/prepared-local.v2.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 83b5ea3c1..43bfa558a 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -2385,7 +2385,8 @@ "response_headers": { "X-Durable-Workflow-Protocol-Version": "$protocol_version", "Content-Type": "application/json media type; unrelated standard headers are not an equality contract" - } + }, + "counter_scope": "preflight lookup/fetch/clock counters begin after the existing independently scoped auth/protocol/runtime-role admission; they count durable task/attempt/claim/registration reads and payload resolution, not source-module loading or pre-existing credential authorization" }, "canonical_projection": { "override": "prepared failed report.next_retry_delay_seconds maps to canonical application failure.next_retry_delay_seconds only", From e29490ed43d4b612b1a2dbe001b36efa6efb32a0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:14:38 +0000 Subject: [PATCH 10/15] Preserve legacy dispatch before local v2 preflight --- docs/contracts/prepared-local-retry-v2.md | 31 +++- .../prepared-local.v2.json | 148 ++++++++++++++++-- 2 files changed, 162 insertions(+), 17 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index b10eab0c0..21d753d5d 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -55,14 +55,29 @@ They retain the distinct controller-missing, controller-stale and structured engine refusals rather than accepting arbitrary `recorded:false` shapes. No new top-level reason alias is selected. -The future handler must run a pure report shape/type/override preflight before -its task, attempt, original-claim or registration lookup. Current Server checks -only that the report is an array before those lookups, so Core's later strict -normalizer alone does not establish this ordering. Recheck admitted shape and -fences under the authoritative transaction. The pure preflight cannot read -storage, fetch payloads, invoke a callback or sample the failure clock. -Core's entire current normalizer cannot be lifted into this phase: its -completed branch resolves payload envelopes. +The future handler uses two phases to select the pure report preflight. +Presence of `report.next_retry_delay_seconds`, including an invalid null value, +signals only untrusted feature intent. After existing auth, protocol and +runtime-role admission, validate that supplied field and its closed failed-report +shape before task, attempt, original-claim or registration reads. A passing +preflight grants no capability, stored mode or snapshot authority. + +Without that member, preserve the legacy lookup and validation order. After +scoped original-claim lookup actually establishes the conjunctive version-2 +mode, run the same pure report validator before fresh registration/fence work, +failure-clock sampling or payload resolution. A malformed legacy report does +not acquire a new early refusal merely by sharing this endpoint. The zero-task- +lookup guarantee therefore applies to malformed supplied overrides. A pure +validator invoked after mode selection has zero reads of its own, not a claim +that the request performed no earlier reads. No new request marker, capability +or global protocol version selects these phases. + +Current Server only checks that the report is an array before lookup. These +phases are future handler requirements. Recheck admitted shape, original mode +and fences under the authoritative transaction. The pure validator cannot read +storage, fetch payloads, invoke a callback or sample the failure clock. Core's +entire current normalizer cannot be lifted into this phase: its completed +branch resolves payload envelopes. The retained fetch-free retry-decision recovery applies to failed and timed-out reports with pure normalized fingerprints. Completed external-reference diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 43bfa558a..f4e7dbe09 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -1727,7 +1727,8 @@ "outcome": "failed", "message": "synthetic", "next_retry_delay_seconds": null - } + }, + "preflight_phase": "new_member_present" }, "expected": { "http_status": 422, @@ -1747,7 +1748,8 @@ "outcome": "failed", "message": "synthetic", "next_retry_delay_seconds": null - } + }, + "preflight_phase": "new_member_present" }, "expected": { "http_status": 422, @@ -1767,7 +1769,8 @@ "outcome": "failed", "message": "synthetic", "next_retry_delay_seconds": null - } + }, + "preflight_phase": "new_member_present" }, "expected": { "http_status": 422, @@ -1787,7 +1790,8 @@ "outcome": "failed", "message": "synthetic", "next_retry_delay_seconds": null - } + }, + "preflight_phase": "new_member_present" }, "expected": { "http_status": 422, @@ -2018,7 +2022,8 @@ "id": "local_v2_shape_preflight_completed_reference_no_fetch", "carrier": { "report": "completed with structurally valid external envelope", - "scoped_claim_authority": "not yet read" + "scoped_claim_authority": "actually read and proves the conjunctive v2 mode", + "preflight_phase": "new_member_absent_proven_v2" }, "expected": { "shape_phase_only": true, @@ -2026,7 +2031,9 @@ "payload_fetch_count": 0, "callback_count": 0, "failure_clock_sample_count": 0, - "payload_authority_or_bytes_qualified": false + "payload_authority_or_bytes_qualified": false, + "counter_scope": "pure validator invocation only; earlier scoped authority reads are outside these counters", + "request_zero_lookup_claim": false } }, { @@ -2118,6 +2125,113 @@ "controller_missing_shape_accepted": false, "setup_or_backend_error_counts_as_pass": false } + }, + { + "id": "local_v2_dispatch_no_member_legacy_missing_precedes_malformed", + "carrier": { + "report": { + "outcome": "failed", + "message": 17 + }, + "next_retry_delay_seconds_member": "absent", + "scoped_task_attempt": "missing", + "original_mode": "not read" + }, + "expected": { + "http_status": 404, + "response_family": "controller_attempt_missing", + "reason": "activity_attempt_not_found", + "early_v2_preflight_count": 0, + "legacy_precedence_preserved": true, + "v2_support_inferred": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_dispatch_no_member_legacy_stale_precedes_malformed", + "carrier": { + "report": { + "outcome": "failed", + "message": 17 + }, + "next_retry_delay_seconds_member": "absent", + "scoped_original_claim": "valid legacy mode", + "fresh_registration": "stale with no retained outcome and no cancellation exception" + }, + "expected": { + "http_status": 409, + "response_family": "controller_registration_stale", + "reason": "stale_worker_registration", + "early_v2_preflight_count": 0, + "legacy_precedence_preserved": true, + "v2_support_inferred": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_dispatch_no_member_v2_after_real_mode", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic" + }, + "next_retry_delay_seconds_member": "absent", + "scoped_original_claim": "actually read and proves every conjunctive v2 gate" + }, + "expected": { + "phase": "new_member_absent_proven_v2", + "pure_validation": "accept shape only after actual mode lookup", + "fresh_registration_or_fence_before_pure_validation": false, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "request_zero_lookup_claim": false, + "mode_from_request_presence": false + } + }, + { + "id": "local_v2_dispatch_no_member_v2_malformed_after_real_mode", + "carrier": { + "report": { + "outcome": "failed", + "message": 17 + }, + "next_retry_delay_seconds_member": "absent", + "scoped_original_claim": "actually read and proves every conjunctive v2 gate" + }, + "expected": { + "phase": "new_member_absent_proven_v2", + "pure_validation": "refuse before fresh registration/fence/clock/payload work", + "request_zero_lookup_claim": false, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0, + "new_http_reason_alias_selected": false + } + }, + { + "id": "local_v2_dispatch_supplied_valid_override_cannot_grant_mode", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": 0 + }, + "phase_a": "pure validation accepts", + "scoped_original_claim": "actual local contract version 1; other required gates valid" + }, + "expected": { + "http_status": 409, + "response_family": "prepared_capability_refusal", + "required_capability": "activity_retry_delay_override_v1", + "unavailable": [ + "local_activity_contract_version" + ], + "v2_support_inferred": false, + "policy_or_clock_selection_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0 + } } ], "prepared_policy_proposal": { @@ -2184,7 +2298,6 @@ "baseline": "valid independent auth/namespace/protocol and original prepared graph; all unrelated gates must be live before an isolated semantic control can qualify", "request_scope": "existing controller outer claim/report validation and normalization remain separate; this proposal closes the nested v2 report and response families, not a new global protocol", "pure_preflight": { - "placement": "after existing auth/protocol/runtime-role admission, before namespace task/attempt/claim/registration lookup", "checks": [ "closed report keys", "actual field types", @@ -2198,7 +2311,24 @@ "callback", "failure-clock sample" ], - "not_reusable_as_is": "Core normalizePortableReport resolves completed payload envelopes before returning; only its pure shape/type interpretation may be shared" + "not_reusable_as_is": "Core normalizePortableReport resolves completed payload envelopes before returning; only its pure shape/type interpretation may be shared", + "dispatch": { + "new_member_present": { + "selector": "report is an object and has its own next_retry_delay_seconds member, including null; untrusted intent only, never mode authority", + "placement": "after existing auth/protocol/runtime-role and outer claim/report validation, before task/attempt/original-claim/registration reads", + "action": "purely validate the supplied override and closed failed-report shape; reject malformed input without durable lookup", + "success": "continue scoped original identity/mode lookup; no v2 support is inferred from presence or validation success" + }, + "new_member_absent": { + "placement": "preserve the existing scoped task/attempt/original-claim lookup order", + "legacy": "retain existing legacy report validation and refusal precedence; do not move its malformed-report check ahead of lookup or registration", + "proven_v2": "only after actual scoped original-claim/mode/snapshot lookup establishes the conjunction, run the same pure closed-report validator before fresh registration/fence/clock/write or payload work" + }, + "authority": "all required capability, contract and snapshot fields must be read and verified from the original scoped authority; unread fields are unknown", + "new_marker_capability_or_protocol": false + }, + "zero_task_lookup_guarantee": "malformed supplied next_retry_delay_seconds and its failed-report shape only; absent-member reports keep mode-dependent lookup/validation precedence", + "function_purity": "both invocations perform zero storage/fetch/callback/failure-clock effects themselves; a post-mode invocation does not erase earlier scoped authority reads" }, "transaction": "recheck admitted shape, immutable original mode/identity, matching retained failed/timeout decision and fresh fences under the existing canonical locks; no controller-only validation grants writer authority", "bindings": { @@ -2386,7 +2516,7 @@ "X-Durable-Workflow-Protocol-Version": "$protocol_version", "Content-Type": "application/json media type; unrelated standard headers are not an equality contract" }, - "counter_scope": "preflight lookup/fetch/clock counters begin after the existing independently scoped auth/protocol/runtime-role admission; they count durable task/attempt/claim/registration reads and payload resolution, not source-module loading or pre-existing credential authorization" + "counter_scope": "after existing independently scoped auth/protocol/runtime-role and outer claim/report validation; malformed supplied overrides count all subsequent task/attempt/claim/registration reads as zero; post-mode pure-validator counters cover only the function invocation and do not claim zero earlier request lookups" }, "canonical_projection": { "override": "prepared failed report.next_retry_delay_seconds maps to canonical application failure.next_retry_delay_seconds only", From c6be3185ad6bd6844c4ba22268c7f1236e0b2e14 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:18:11 +0000 Subject: [PATCH 11/15] Bind local report refusal families and validation order --- docs/contracts/prepared-local-retry-v2.md | 11 +- .../prepared-local.v2.json | 122 +++++++++++++++++- 2 files changed, 127 insertions(+), 6 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index 21d753d5d..8619ac9f9 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -60,14 +60,19 @@ Presence of `report.next_retry_delay_seconds`, including an invalid null value, signals only untrusted feature intent. After existing auth, protocol and runtime-role admission, validate that supplied field and its closed failed-report shape before task, attempt, original-claim or registration reads. A passing -preflight grants no capability, stored mode or snapshot authority. +preflight grants no capability, stored mode or snapshot authority. Check override +value/type and failed-only placement first. An invalid or forbidden override +uses the proposed strict `invalid_activity_retry_delay` 422 reply. Other closed +report errors use the existing `invalid_local_activity_outcome` engine-refusal +409 reply, including a valid supplied override beside an invalid message or +unknown report member. Neither refusal requires a durable lookup in this phase. Without that member, preserve the legacy lookup and validation order. After scoped original-claim lookup actually establishes the conjunctive version-2 mode, run the same pure report validator before fresh registration/fence work, failure-clock sampling or payload resolution. A malformed legacy report does -not acquire a new early refusal merely by sharing this endpoint. The zero-task- -lookup guarantee therefore applies to malformed supplied overrides. A pure +not acquire a new early refusal merely by sharing this endpoint. The guarantee of no task lookups therefore applies to refusals in the +new-member phase. A pure validator invoked after mode selection has zero reads of its own, not a claim that the request performed no earlier reads. No new request marker, capability or global protocol version selects these phases. diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index f4e7dbe09..076a2f0b8 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -2206,7 +2206,10 @@ "failure_clock_sample_count": 0, "durable_mutation_count": 0, "next_callback_count": 0, - "new_http_reason_alias_selected": false + "new_http_reason_alias_selected": false, + "http_status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome" } }, { @@ -2232,6 +2235,101 @@ "durable_mutation_count": 0, "next_callback_count": 0 } + }, + { + "id": "local_v2_http_valid_override_invalid_message", + "carrier": { + "report": { + "outcome": "failed", + "message": 17, + "next_retry_delay_seconds": 0 + }, + "preflight_phase": "new_member_present", + "scoped_original_mode": "unread and unknown" + }, + "expected": { + "http_status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0, + "v2_support_inferred": false + } + }, + { + "id": "local_v2_http_valid_override_unknown_report_member", + "carrier": { + "report": { + "outcome": "failed", + "message": "synthetic", + "next_retry_delay_seconds": 0, + "unknown_report_member": true + }, + "preflight_phase": "new_member_present", + "scoped_original_mode": "unread and unknown" + }, + "expected": { + "http_status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0, + "v2_support_inferred": false + } + }, + { + "id": "local_v2_http_invalid_override_and_invalid_message", + "carrier": { + "report": { + "outcome": "failed", + "message": 17, + "next_retry_delay_seconds": null + }, + "preflight_phase": "new_member_present", + "scoped_original_mode": "unread and unknown" + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "reason": "invalid_activity_retry_delay", + "generic_report_error_preempts_override": false, + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0, + "v2_support_inferred": false + } + }, + { + "id": "local_v2_http_forbidden_override_and_invalid_completed_result", + "carrier": { + "report": { + "outcome": "completed", + "result": 17, + "next_retry_delay_seconds": 0 + }, + "preflight_phase": "new_member_present", + "scoped_original_mode": "unread and unknown" + }, + "expected": { + "http_status": 422, + "response_family": "strict_override_refusal", + "reason": "invalid_activity_retry_delay", + "generic_report_error_preempts_override": false, + "storage_lookup_count": 0, + "payload_fetch_count": 0, + "failure_clock_sample_count": 0, + "durable_mutation_count": 0, + "next_callback_count": 0, + "v2_support_inferred": false + } } ], "prepared_policy_proposal": { @@ -2327,8 +2425,26 @@ "authority": "all required capability, contract and snapshot fields must be read and verified from the original scoped authority; unread fields are unknown", "new_marker_capability_or_protocol": false }, - "zero_task_lookup_guarantee": "malformed supplied next_retry_delay_seconds and its failed-report shape only; absent-member reports keep mode-dependent lookup/validation precedence", - "function_purity": "both invocations perform zero storage/fetch/callback/failure-clock effects themselves; a post-mode invocation does not erase earlier scoped authority reads" + "zero_task_lookup_guarantee": "refusals in the new-member-present phase, including strict override and remaining report-shape failures; absent-member reports keep mode-dependent lookup/validation precedence", + "function_purity": "both invocations perform zero storage/fetch/callback/failure-clock effects themselves; a post-mode invocation does not erase earlier scoped authority reads", + "error_precedence": [ + { + "check": "a supplied next_retry_delay_seconds has an actual integer in the portable range and outcome is failed", + "failure": { + "status": 422, + "response_family": "strict_override_refusal", + "reason": "invalid_activity_retry_delay" + } + }, + { + "check": "remaining closed report keys, required members/types and inline/reference envelope structural form", + "failure": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome" + } + } + ] }, "transaction": "recheck admitted shape, immutable original mode/identity, matching retained failed/timeout decision and fresh fences under the existing canonical locks; no controller-only validation grants writer authority", "bindings": { From e974e28550ab5d06a26ee01ad1c5a95973327920 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:20:51 +0000 Subject: [PATCH 12/15] Bind malformed JSON diagnostics to exact source cases --- .../prepared-local.v2.json | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 076a2f0b8..cdcfdb432 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -2071,7 +2071,8 @@ "http_status": 400, "response_family": "native_malformed_json", "typed_override_verdict": false, - "durable_mutation_count": 0 + "durable_mutation_count": 0, + "json_error": "Syntax error" } }, { @@ -2084,7 +2085,8 @@ "http_status": 400, "response_family": "native_malformed_json", "typed_override_verdict": false, - "durable_mutation_count": 0 + "durable_mutation_count": 0, + "json_error": "Syntax error" } }, { @@ -2097,7 +2099,8 @@ "http_status": 400, "response_family": "native_malformed_json", "typed_override_verdict": false, - "durable_mutation_count": 0 + "durable_mutation_count": 0, + "json_error": "Syntax error" } }, { @@ -2109,7 +2112,8 @@ "http_status": 400, "response_family": "native_malformed_json", "typed_override_verdict": false, - "durable_mutation_count": 0 + "durable_mutation_count": 0, + "json_error": "Malformed UTF-8 characters, possibly incorrectly encoded" } }, { @@ -2391,6 +2395,10 @@ "native_json": { "path": "rust/src/runtime/json_body.rs", "blob": "50664ec66ce1cee1c8ecaf1d5a2d52a2f21a772c" + }, + "native_payload_limits": { + "path": "rust/src/runtime/payload_limits.rs", + "blob": "d0bfeef89cd9a280c9e478ee62125fbaa01ef78d" } }, "baseline": "valid independent auth/namespace/protocol and original prepared graph; all unrelated gates must be live before an isolated semantic control can qualify", @@ -2467,7 +2475,8 @@ "$required_capability": "prepared_local_activities for existing role/original-prepared-claim refusal; activity_retry_delay_override_v1 only for proposed fresh v2 mode-gate refusal", "$unavailable": "the exact ordered array explicitly chosen by the case; no wildcard members", "$duplicate_boolean": "actual Boolean required by the case, false for first commit and true for identical retained reply", - "$case_claim_released_boolean": "actual Boolean, true only for ActivityRetryScheduled, false for terminal/completed" + "$case_claim_released_boolean": "actual Boolean, true only for ActivityRetryScheduled, false for terminal/completed", + "$case_json_error": "only the fixed case.expected.json_error literal, independently derived from pinned payload_limits::json_error; never copied from a response or generalized to another malformed input" }, "matching": "resolve declared bindings only; compare full exact JSON keys/types/values and the protocol response header; response decoding must consume strict UTF-8 original bytes fully and refuse duplicate member names before schema matching", "envelope": { @@ -2614,11 +2623,15 @@ "body": { "message": "Request bodies must contain valid JSON.", "reason": "malformed_json", - "json_error": "Syntax error", + "json_error": "$case_json_error", "protocol_version": "$protocol_version", "server_capabilities": "$server_capabilities" }, - "typed_override_verdict": false + "typed_override_verdict": false, + "closed_case_json_error_values": [ + "Syntax error", + "Malformed UTF-8 characters, possibly incorrectly encoded" + ] } }, "mode_gate_unavailable_proposal": { From 66ae0e3ae3cb87ecdbf01f279f3e131268089f2a Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:27:20 +0000 Subject: [PATCH 13/15] docs: record reviewed prepared-retry baseline and concrete writer dependency --- docs/contracts/prepared-local-retry-v2.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index 8619ac9f9..04905d1bf 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -1,7 +1,9 @@ # Prepared local retry contract proposal -This is an unfrozen source proposal for the local part of organization issue -[#219](https://github.com/durable-workflow/.github/issues/219). The machine-readable +This is the reviewed semantic source baseline for the local part of organization +issue [#219](https://github.com/durable-workflow/.github/issues/219). General recorded +the [implementation and adoption order](https://github.com/durable-workflow/.github/issues/219#issuecomment-6671268805) +after the joint source comparison. The machine-readable companion is [prepared-local.v2.json](../../tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json). It adds no runtime validator, producer, advertisement or executing test result. The existing 54 canonical policy cases and 37 remote adapter cases are unchanged. @@ -50,7 +52,8 @@ matching receipt. It is not accepted worker input. Existing receipt identities, claim-release behavior and created-task identity remain authoritative. The companion now also proposes complete HTTP response families, the fixed zero-history cursor, pinned protocol envelope and finite status/refusal cases. -Those templates still need joint source review and an actual future adapter. +Core completed the [bounded joint source comparison](https://github.com/durable-workflow/server/pull/1034#issuecomment-6671101501). +These templates still require an actual future adapter. They retain the distinct controller-missing, controller-stale and structured engine refusals rather than accepting arbitrary `recorded:false` shapes. No new top-level reason alias is selected. @@ -106,7 +109,13 @@ its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. Workflow owns the complete normalized snapshot and existing timeout/range binding. Server owns the companion validator, transaction, transport and storage -bindings. The exact output object, final normalized ranges -and HTTP bindings must be reconciled before these bytes are frozen or SDKs -adopt them. Native implementation, schema, installed/backend qualification and -release remain separate work. +bindings. The reviewed output object, normalized ranges and HTTP bindings are the source +baseline for the guarded implementation. SDK adoption requires concrete producer +capability and receipt evidence; a remote capability alone does not supply local +contract-2 authority. Core +[#1051](https://github.com/durable-workflow/workflow/pull/1051) owns its kernel and +optional installed bridge role. Server must bind that actual role and its +canonical original mode/snapshot to the retained claim under the existing outer +transaction. The current bound bridge does not expose that role. Native +implementation, schema, installed/backend qualification and release remain +separate work. From a983d76b177d30381e49fac702992b7704ec6744 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:14:09 +0000 Subject: [PATCH 14/15] docs: bind fenced engine interruption to the prepared retry companion --- docs/contracts/prepared-local-retry-v2.md | 31 + .../prepared-local.v2.json | 734 +++++++++++++++++- 2 files changed, 753 insertions(+), 12 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index 04905d1bf..f06d372fb 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -107,6 +107,37 @@ actual backend duration or timestamp bound. Neither the old 100-report batch limit nor its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. +The engine-interruption extension follows General's +[bounded source choice](https://github.com/durable-workflow/.github/issues/219#issuecomment-6674785967). +It appends 24 proposed recovery controls while preserving all 133 reviewed cases. +Only an engine recovery that proves loss or fencing of the original claim may +retain `failure_source: engine_interruption`. Its clock is the first locked +recovery observation that commits. An actually due activity timeout takes the +existing engine-timeout branch instead. Interruption uses the retained policy, +attempt budget, cap and original deadline, with no requested delay or timeout +kind. An ordinary valid failed report remains an application failure; a worker +cannot select recovery provenance or supply engine decision or stop metadata. + +Recovery retains its existing `recovered` receipt carrier, which differs from an +outcome's `recorded` receipt. The proposed version-2 recovery response adds the +retained `retry_disposition` to that closed carrier and its existing cursor and +protocol envelope. Its task identity is the recovering task, independently +bound to the original prepared graph. The existing `local_recovery.version: 1` +identity envelope does not grant local contract 2; that authority still comes +from the canonical original preparation, snapshot and retained claim. Version-1 +recovery bytes are unchanged. + +A matching committed recovery receipt returns its original sample and decision +before fresh fence or clock calculations. Rollback publishes no receipt, +decision, task or event. A terminal decision creates no following attempt; a +scheduled decision still requires independent retry preparation and callback +admission. Recovery and lease fencing do not prove physical callback stop or a +supervisor join. The immutable recovery receipt's `callback_stop_state` stays +`unknown`, including after a separate real stop acknowledgement updates other +state. These guards and the unadvertised installed role remain in force. The +extension requires cross-owner shape reconciliation before a shared freeze; +the appended data is not an executing recovery result. + Workflow owns the complete normalized snapshot and existing timeout/range binding. Server owns the companion validator, transaction, transport and storage bindings. The reviewed output object, normalized ranges and HTTP bindings are the source diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index cdcfdb432..5135e0b4e 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -1,6 +1,6 @@ { "schema": "durable-workflow.prepared-local-retry-companion.v1", - "status": "unfrozen source proposal; no validator, writer or executing result", + "status": "reviewed 133-row baseline with proposed engine-interruption extension; no validator, writer or executing result", "runtime_enabled": false, "authority": { "scope": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6668566058", @@ -24,10 +24,32 @@ }, "prior_rows_mutated": false, "prepared_timeout_projection": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6669526988", - "core_snapshot_range_proposal": "https://github.com/durable-workflow/workflow/pull/1038#issuecomment-6669717376" + "core_snapshot_range_proposal": "https://github.com/durable-workflow/workflow/pull/1038#issuecomment-6669717376", + "engine_interruption": { + "selection": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6674785967", + "core_proposal": "https://github.com/durable-workflow/.github/issues/219#issuecomment-6673550956", + "existing_carrier_source": { + "commit": "4b2cdc5507120d9c213b0b02a85738bec96c574a", + "executor": { + "path": "src/V2/Support/LocalActivityExecutor.php", + "blob": "4398d6e705c8e1bdece33d2734ef0d868cb5bebb" + }, + "preparation": { + "path": "src/V2/Support/PortableLocalActivityPreparation.php", + "blob": "07d3f302159293c1785a99ba50aeb2512e86c6b5" + }, + "runtime": { + "path": "src/V2/Support/LocalActivityRuntime.php", + "blob": "49e7e3d7574fafd9d7246c97932d49f9cbb8fba9" + } + }, + "scope": "existing v1 recovery/receipt source facts only; not a contract-2 writer or installed-role qualification", + "prior_133_cases_mutated": false, + "extension_frozen": false + } }, "adoption_gates": [ - "Core accepts the proposed closed retry_disposition fields and normalized snapshot/range binding", + "Core/Server reconcile this engine-interruption extension over the reviewed closed 133-row baseline before its writer/consumer adoption", "Core/Server reconcile the proposed HTTP families, pure preflight placement and separate completed payload scope before final source freeze", "Native implements the prepared role and validator/transaction writer before advertisement", "Actual native/database/SDK/installed qualification and release remain separate" @@ -89,7 +111,12 @@ "retry_available_at", "failure_time", "original_schedule_to_close_deadline_at", - "snapshot_version" + "snapshot_version", + "failure_source", + "requested_delay_seconds", + "timeout_kind", + "local_recovery", + "callback_stop_state" ] }, "clock": { @@ -111,7 +138,9 @@ "fresh original claim and temporal fences using one sample", "actually due engine timeout", "application failure retry selection" - ] + ], + "recovery_sample": "for a fresh proven contract-2 recovery, the first locked engine observation that commits; never a claimed callback failure time or original lease-expiry timestamp", + "recovery_duplicate": "after scoped immutable identity/mode validation, return the matching committed recovery receipt and decision before any fresh clock, fence or policy calculation; rollback publishes none" }, "retry_disposition_proposal": { "boundary": "engine-owned object shared exactly by the stored outcome metadata and its prepared receipt; never accepted in a worker report", @@ -121,7 +150,7 @@ "decision": "scheduled or terminal", "failure_time": "the stored first accepted engine sample", "original_schedule_to_close_deadline_at": "stored UTC timestamp or null", - "failure_source": "application or engine_timeout" + "failure_source": "application or engine_timeout or engine_interruption" }, "common_optional": { "requested_delay_seconds": "present iff the valid failed report supplied next_retry_delay_seconds; exact requested integer, including zero; diagnostic input does not prove application" @@ -130,7 +159,7 @@ "candidate_delay_seconds": "selected nonnegative portable integer", "maximum_interval_seconds": "finite cap from retained snapshot 2", "retry_available_at": "checked failure_time plus selected seconds, strictly before original deadline if set", - "delay_source": "policy or failure_override; engine timeouts always policy" + "delay_source": "policy or failure_override; engine timeouts and engine interruptions always policy" }, "scheduled_forbidden": [ "terminal_classification" @@ -168,7 +197,8 @@ ], "unknown_members": "refused by a future consumer; never silently ignored", "conditional_common": { - "timeout_kind": "required iff failure_source is engine_timeout; one of start_to_close, schedule_to_close, heartbeat; forbidden otherwise" + "timeout_kind": "required iff failure_source is engine_timeout; one of start_to_close, schedule_to_close, heartbeat; forbidden otherwise", + "engine_interruption": "only a fenced engine recovery branch; requested_delay_seconds and timeout_kind forbidden; any selected delay_source must be policy" }, "delay_source_rule": "failure_override only for a still-admitted application failure with a supplied valid override; otherwise policy. A preempted application override stays requested diagnostic input only.", "engine_timeout_terminal_classification": "reserved for overall schedule_to_close expiry; start_to_close/heartbeat before the total deadline may schedule the policy retry or terminate under their actual budget/cap/deadline branch" @@ -220,7 +250,8 @@ "created_task_ids": [], "retry_disposition": "absent" }, - "http_boundary": "the proposed full reply envelopes and distinct refusal families are declared in http_proposal; they still require actual future handler/adapter implementation and qualification" + "http_boundary": "the proposed full reply envelopes and distinct refusal families are declared in http_proposal; they still require actual future handler/adapter implementation and qualification", + "boundary": "existing_required and outcome variants describe outcome receipts only; recovery retains its distinct existing recovered carrier declared in recovery_proposal" }, "history": { "identity": "same run/execution/real attempt/worker attempt/original task epoch as the prepared outcome receipt", @@ -228,7 +259,10 @@ "terminal": "one existing ActivityFailed or ActivityTimedOut outcome for the real attempt; matching retry_disposition; no retry task, following attempt/start or ActivityRetryScheduled", "failure_preservation": "cap/deadline retry termination preserves the admitted application failure/type; already-due engine timeout preserves engine timeout provenance instead", "cold_replay": "consume stored failed/timeout retry disposition and availability; never evaluate current policy/clock for that decision; no fetch-free completed replay claim", - "timeout_scheduled": "an actual pre-total-deadline start_to_close/heartbeat timeout may record ActivityRetryScheduled with retry_reason timeout, actual timeout_kind and policy-selected delay; no worker override is applied" + "timeout_scheduled": "an actual pre-total-deadline start_to_close/heartbeat timeout may record ActivityRetryScheduled with retry_reason timeout, actual timeout_kind and policy-selected delay; no worker override is applied", + "interruption": "retain local_recovery and the engine RuntimeException provenance; scheduled uses the existing cold-replay retry reason; terminal uses ActivityFailed, except that an actually due timeout retains engine_timeout precedence", + "recovery_replay": "consume the stored recovery decision and original unknown stop state without current clock/policy/physical-state substitution; no callback is invoked by recovery", + "interruption_retry_reason": "cold_replay" }, "ranges": { "portable_integer_minimum": 0, @@ -2334,6 +2368,587 @@ "next_callback_count": 0, "v2_support_inferred": false } + }, + { + "id": "local_v2_recovery_policy_retry", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "scheduled", + "delay_source": "policy", + "candidate_delay_seconds": 5, + "retry_available_at_seconds": 95, + "event_type": "ActivityRetryScheduled", + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "recovered": true, + "status": 200, + "response_family": "recovered_retry_decision", + "retry_reason": "cold_replay", + "maximum_interval_seconds": 20 + } + }, + { + "id": "local_v2_recovery_zero_policy_retry", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 0, + "maximum_interval_seconds": 0 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "scheduled", + "delay_source": "policy", + "candidate_delay_seconds": 0, + "retry_available_at_seconds": 90, + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "retry_reason": "cold_replay", + "maximum_interval_seconds": 0 + } + }, + { + "id": "local_v2_recovery_policy_non_retryable", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20, + "snapshot_non_retryable_error_types": [ + "RuntimeException" + ] + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "terminal", + "terminal_classification": "non_retryable", + "candidate_delay_member": "absent", + "delay_source_member": "absent", + "event_type": "ActivityFailed", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "maximum_interval_member": "absent" + } + }, + { + "id": "local_v2_recovery_attempts_exhausted", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 3, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "terminal", + "terminal_classification": "attempts_exhausted", + "candidate_delay_member": "absent", + "delay_source_member": "absent", + "event_type": "ActivityFailed", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "maximum_interval_member": "absent" + } + }, + { + "id": "local_v2_recovery_terminal_cap", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 21, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_policy_cap", + "delay_source": "policy", + "candidate_delay_seconds": 21, + "retry_available_at_member": "absent", + "event_type": "ActivityFailed", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "maximum_interval_seconds": 20 + } + }, + { + "id": "local_v2_recovery_terminal_deadline", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 10, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "decision": "terminal", + "terminal_classification": "retry_delay_exceeds_schedule_to_close", + "delay_source": "policy", + "candidate_delay_seconds": 10, + "retry_available_at_member": "absent", + "event_type": "ActivityFailed", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "maximum_interval_seconds": 20 + } + }, + { + "id": "local_v2_recovery_due_overall_timeout", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 100, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": "schedule_to_close", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_timeout", + "timeout_kind": "schedule_to_close", + "decision": "terminal", + "terminal_classification": "engine_timeout", + "requested_delay_member": "absent", + "event_type": "ActivityTimedOut", + "callback_stop_state": "unknown", + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "interruption_selection_count": 0 + } + }, + { + "id": "local_v2_recovery_due_start_timeout_policy_retry", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": "start_to_close", + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "expected": { + "failure_source": "engine_timeout", + "timeout_kind": "start_to_close", + "decision": "scheduled", + "delay_source": "policy", + "candidate_delay_seconds": 5, + "retry_available_at_seconds": 95, + "requested_delay_member": "absent", + "callback_stop_state": "unknown", + "new_retry_task_count": 1, + "new_retry_event_count": 1, + "new_attempt_count": 0, + "interruption_selection_count": 0, + "retry_reason": "timeout" + } + }, + { + "id": "local_v2_recovery_duplicate_after_deadline", + "carrier": { + "first_recovery": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "first_transaction": "committed", + "original_reply": "lost", + "identical_duplicate_sample_seconds": 110, + "current_policy": "changed", + "separate_stop_acknowledgement": "arrived after the committed recovery" + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "duplicate": true, + "retained_failure_time_seconds": 90, + "retained_retry_available_at_seconds": 95, + "retained_decision_unchanged": true, + "fresh_clock_sample_count": 0, + "fresh_fence_calculation_count": 0, + "new_retry_task_count": 0, + "new_retry_event_count": 0 + } + }, + { + "id": "local_v2_recovery_rollback", + "carrier": { + "fresh_recovery": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "transaction": "rollback" + }, + "expected": { + "published_sample_count": 0, + "published_decision_count": 0, + "published_receipt_count": 0, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "durable_attempt_status_change_count": 0 + } + }, + { + "id": "local_v2_recovery_original_claim_still_live", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "original different hosting task remains live", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "expected": { + "reason": "local_activity_original_claim_live", + "engine_interruption_decision_count": 0, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_foreign_retained_receipt", + "carrier": { + "recovery_identity": "otherwise valid scoped original graph", + "retained_receipt": "descriptor fingerprint alone differs; every run/attempt/task identity remains matched" + }, + "expected": { + "reason": "local_activity_recovery_mismatch", + "retained_receipt_returned": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_worker_failure_source", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "failed", + "message": "synthetic failure", + "failure_source": "engine_interruption" + } + }, + "expected": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "engine_interruption_authority_granted": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_worker_outcome", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "engine_interruption" + } + }, + "expected": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "engine_interruption_authority_granted": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_worker_requested_delay", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "failed", + "message": "synthetic failure", + "requested_delay_seconds": 1 + } + }, + "expected": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "engine_interruption_authority_granted": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_worker_timeout_kind", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "failed", + "message": "synthetic failure", + "timeout_kind": "heartbeat" + } + }, + "expected": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "engine_interruption_authority_granted": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_worker_stop_state", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "failed", + "message": "synthetic failure", + "callback_stop_state": "unknown" + } + }, + "expected": { + "status": 409, + "response_family": "engine_refusal", + "reason": "invalid_local_activity_outcome", + "engine_interruption_authority_granted": false, + "durable_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_output_requested_delay", + "carrier": { + "otherwise_valid_engine_interruption_output": true, + "isolated_extra_member": { + "requested_delay_seconds": 1 + } + }, + "expected": { + "consumer_verdict": "refuse", + "qualified_engine_decision": false + } + }, + { + "id": "local_v2_recovery_output_timeout_kind", + "carrier": { + "otherwise_valid_engine_interruption_output": true, + "isolated_extra_member": { + "timeout_kind": "heartbeat" + } + }, + "expected": { + "consumer_verdict": "refuse", + "qualified_engine_decision": false + } + }, + { + "id": "local_v2_recovery_output_stop_state_rewritten", + "carrier": { + "original_committed_recovery_receipt": "unknown", + "after_separate_real_stop_acknowledgement": true, + "duplicate_receipt_callback_stop_state": "stopped" + }, + "expected": { + "consumer_verdict": "refuse", + "retained_recovery_receipt_mutation_count": 0 + } + }, + { + "id": "local_v2_recovery_ordinary_failed_override_remains_application", + "carrier": { + "operation": "prepared_outcome", + "report": { + "outcome": "failed", + "message": "synthetic failure", + "next_retry_delay_seconds": 1 + }, + "original_claim_live": true + }, + "expected": { + "failure_source": "application", + "delay_source": "failure_override", + "engine_interruption_authority_granted": false + } + }, + { + "id": "local_v2_recovery_v1_remains_v1", + "carrier": { + "original_local_contract_version": 1, + "recovery_identity_envelope_version": 1, + "otherwise_plausible_policy": true + }, + "expected": { + "local_v2_authority": "unsupported", + "legacy_recovery_and_history_bytes": "unchanged", + "v2_relabelling_count": 0 + } + }, + { + "id": "local_v2_recovery_first_locked_observation_not_lease_expiry", + "carrier": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20, + "original_lease_expiry_seconds": 20, + "claimed_worker_failure_seconds": 30 + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "failure_time_seconds": 90, + "decision": "scheduled", + "retry_available_at_seconds": 95, + "selected_sample_count": 1 + } + }, + { + "id": "local_v2_recovery_concurrent_identical_recovery", + "carrier": { + "fresh_recovery": { + "operation": "prepared_recovery", + "original_mode": "proven contract 2 and retained snapshot 2", + "previous_claim_loss": "independently proven under canonical locks", + "current_recovery_claim": "independently scoped and valid", + "engine_sample_seconds": 90, + "original_deadline_seconds": 100, + "actually_due_timeout_kind": null, + "attempt_number": 1, + "max_attempts": 3, + "policy_delay_seconds": 5, + "maximum_interval_seconds": 20 + }, + "requests": "two identical scoped recovery operations under canonical serialization" + }, + "expected": { + "failure_source": "engine_interruption", + "callback_stop_state": "unknown", + "requested_delay_member": "absent", + "timeout_kind_member": "absent", + "new_attempt_count": 0, + "callback_invocation_count": 0, + "committed_decision_count": 1, + "committed_receipt_count": 1, + "first_duplicate": false, + "second_duplicate": true, + "published_sample_count": 1, + "new_retry_task_count": 1, + "new_retry_event_count": 1 + } } ], "prepared_policy_proposal": { @@ -2476,7 +3091,8 @@ "$unavailable": "the exact ordered array explicitly chosen by the case; no wildcard members", "$duplicate_boolean": "actual Boolean required by the case, false for first commit and true for identical retained reply", "$case_claim_released_boolean": "actual Boolean, true only for ActivityRetryScheduled, false for terminal/completed", - "$case_json_error": "only the fixed case.expected.json_error literal, independently derived from pinned payload_limits::json_error; never copied from a response or generalized to another malformed input" + "$case_json_error": "only the fixed case.expected.json_error literal, independently derived from pinned payload_limits::json_error; never copied from a response or generalized to another malformed input", + "$recovery_workflow_task_id": "the independently authorized recovering task ID from the matching recovery event, distinct from the original prepared hosting task" }, "matching": "resolve declared bindings only; compare full exact JSON keys/types/values and the protocol response header; response decoding must consume strict UTF-8 original bytes fully and refuse duplicate member names before schema matching", "envelope": { @@ -2632,6 +3248,29 @@ "Syntax error", "Malformed UTF-8 characters, possibly incorrectly encoded" ] + }, + "recovered_retry_decision": { + "status": 200, + "body": { + "recovered": true, + "duplicate": "$duplicate_boolean", + "reason": null, + "event_id": "$event_id", + "event_type": "$event_type", + "activity_execution_id": "$activity_execution_id", + "activity_attempt_id": "$activity_attempt_id", + "workflow_task_id": "$recovery_workflow_task_id", + "recorded_at": "$recorded_at", + "callback_stop_state": "unknown", + "claim_released": "$case_claim_released_boolean", + "created_task_ids": "$created_task_ids", + "history_refresh_page_token": "MA==", + "protocol_version": "$protocol_version", + "server_capabilities": "$server_capabilities", + "retry_disposition": "$retry_disposition" + }, + "recorded_member": "absent, never a recorded/recovered alias", + "binding_scope": "existing canonical recovery receipt validation, with v2 event metadata/disposition equality; no new worker-report authority" } }, "mode_gate_unavailable_proposal": { @@ -2645,12 +3284,83 @@ "X-Durable-Workflow-Protocol-Version": "$protocol_version", "Content-Type": "application/json media type; unrelated standard headers are not an equality contract" }, - "counter_scope": "after existing independently scoped auth/protocol/runtime-role and outer claim/report validation; malformed supplied overrides count all subsequent task/attempt/claim/registration reads as zero; post-mode pure-validator counters cover only the function invocation and do not claim zero earlier request lookups" + "counter_scope": "after existing independently scoped auth/protocol/runtime-role and outer claim/report validation; malformed supplied overrides count all subsequent task/attempt/claim/registration reads as zero; post-mode pure-validator counters cover only the function invocation and do not claim zero earlier request lookups", + "recovery_scope": "the existing recover route keeps its recovered carrier; this v2 extension adds a retained retry_disposition only after actual original mode/claim authority. Outcome report refusals and the nine original families are unchanged" }, "canonical_projection": { "override": "prepared failed report.next_retry_delay_seconds maps to canonical application failure.next_retry_delay_seconds only", "failure_type": "the normalized admitted application exception_type; engine timeouts retain their own source/type", "time": "canonical whole-second examples map into the single stored authoritative UTC microsecond sample and immutable deadline; do not truncate subsecond comparisons", "preempted_row": "zero_at_original_deadline_is_terminal remains an abstract selector row, preempted by actual overall engine timeout in the prepared HTTP path" + }, + "recovery_proposal": { + "boundary": "existing prepared recover operation and canonical engine primitive, not a new worker-report outcome or global interface", + "authority": "existing scoped current recovery claim, canonical original Started/preparation, actual loss/fencing of the previous claim, stored contract 2 plus snapshot 2 and retained original claim capability; request flags and unread fields grant nothing", + "precedence": [ + "independently scoped auth/protocol/role and strict input/mode/identity validation", + "matching immutable committed v2 recovery receipt", + "fresh canonical current-recovery and previous-claim-loss fences using one engine sample", + "actually due activity timeout under the existing timeout-kind precedence", + "engine-observed interruption selected only when no activity timeout is due" + ], + "interruption_exception": "the existing engine-generated RuntimeException and message from recordInterruptedAttempt; never a worker-supplied exception, failure timestamp or timeout kind", + "policy": "retained snapshot 2 only, including non-retryable type, attempt budget, finite cap, checked availability and original deadline; no cap clamp, v1 fallback or batch-clock substitution", + "worker_input": "no new worker-report member; engine_interruption is forbidden as report.outcome or report.failure_source, and engine output/requested-delay/timeout/stop metadata remains forbidden. A valid ordinary failed override remains an application report, never recovery provenance", + "local_recovery": { + "placement": "unchanged event payload.local_recovery, sibling of retry_disposition; never accepted from a worker", + "required": { + "version": 1, + "descriptor_fingerprint": "original normalized descriptor fingerprint", + "workflow_task_id": "current independently authorized recovery task ID", + "workflow_task_attempt": "current recovery task epoch integer", + "lease_owner": "current recovery lease owner", + "original_workflow_task_id": "original prepared task ID", + "original_workflow_task_attempt": "original prepared task epoch integer", + "original_lease_owner": "original attempt lease owner", + "original_lease_expires_at": "original retained attempt lease expiry", + "callback_stop_state": "unknown" + }, + "optional": [], + "unknown_members": "refuse in a future v2 validator", + "version_scope": "the existing recovery identity-envelope version is not local_activity_contract_version; it supplies no contract-2 authority and v1 events remain byte-unchanged" + }, + "receipt": { + "existing_required": [ + "recovered", + "duplicate", + "reason", + "event_id", + "event_type", + "activity_execution_id", + "activity_attempt_id", + "workflow_task_id", + "recorded_at", + "callback_stop_state", + "claim_released", + "created_task_ids" + ], + "new_required_for_v2": [ + "retry_disposition" + ], + "recovered": true, + "reason": null, + "callback_stop_state": "unknown", + "event_type": "ActivityRetryScheduled for scheduled; ActivityFailed for terminal interruption; existing ActivityRetryScheduled/ActivityTimedOut timeout branches when a timeout is due", + "claim_released": "true iff the retained event is ActivityRetryScheduled", + "created_task_ids": "exactly the retained single retry task ID for scheduled, empty for terminal", + "workflow_task_id": "the recovering task, not a substituted original task", + "forbidden": [ + "recorded", + "workflow_run_id", + "workflow_task_attempt", + "worker_attempt_id" + ], + "unknown_members": "refuse in the complete future v2 recovery reply after its existing protocol/cursor envelope" + }, + "identity": "receipt/event must bind the same original run, execution, real attempt, worker-attempt identity, original task epoch and descriptor, plus the distinct current recovery task/epoch/owner; absence of a top-level reply field is not permission to omit the canonical binding", + "physical_state": "lease fencing and an engine recovery event prove neither physical callback stop nor supervisor join; this immutable recovery metadata/receipt stays unknown even after a separate real stop acknowledgement changes other state", + "retry_preparation": "a scheduled recovery creates one retry task/event and zero following attempts; actual callback admission/preparation retains all existing independent physical/admission/claim guards, retained availability and the original total deadline", + "legacy": "all original v1 recovery, zero-tail/history and corpus bytes remain unchanged; a recovery envelope version of 1 cannot relabel a v1 preparation as contract 2", + "execution": "all appended cases are source requirements, not executing results; no role, capability, SDK adoption or operation is enabled" } } From abe76940450746b5f3691e40958ef4d2cef536c4 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Fri, 6 Nov 2026 02:50:52 +0000 Subject: [PATCH 15/15] docs: distinguish corrupt recovery snapshots from valid policy decisions --- docs/contracts/prepared-local-retry-v2.md | 13 +++- .../prepared-local.v2.json | 74 +++++++++++++------ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/docs/contracts/prepared-local-retry-v2.md b/docs/contracts/prepared-local-retry-v2.md index f06d372fb..89e17fe13 100644 --- a/docs/contracts/prepared-local-retry-v2.md +++ b/docs/contracts/prepared-local-retry-v2.md @@ -109,13 +109,16 @@ its 86,400-second executed-backoff ceiling becomes an implicit version-2 cap. The engine-interruption extension follows General's [bounded source choice](https://github.com/durable-workflow/.github/issues/219#issuecomment-6674785967). -It appends 24 proposed recovery controls while preserving all 133 reviewed cases. +It appends 25 proposed recovery controls while preserving all 133 reviewed cases. Only an engine recovery that proves loss or fencing of the original claim may retain `failure_source: engine_interruption`. Its clock is the first locked recovery observation that commits. An actually due activity timeout takes the existing engine-timeout branch instead. Interruption uses the retained policy, attempt budget, cap and original deadline, with no requested delay or timeout -kind. An ordinary valid failed report remains an application failure; a worker +kind. Valid normalized policy delays already fit their cap; a corrupt retained +cap/delay relationship refuses without a decision or mutation, while the +existing application-override cap cases retain their separate behavior. An +ordinary valid failed report remains an application failure; a worker cannot select recovery provenance or supply engine decision or stop metadata. Recovery retains its existing `recovered` receipt carrier, which differs from an @@ -128,7 +131,11 @@ from the canonical original preparation, snapshot and retained claim. Version-1 recovery bytes are unchanged. A matching committed recovery receipt returns its original sample and decision -before fresh fence or clock calculations. Rollback publishes no receipt, +before fresh fence or clock calculations. Later configuration for new authoring +may differ, while the original retained snapshot, request descriptor and Started +authority remain exact. A changed canonical retained graph refuses even when a +committed receipt exists; replay does not repair or replace that authority. +Rollback publishes no receipt, decision, task or event. A terminal decision creates no following attempt; a scheduled decision still requires independent retry preparation and callback admission. Recovery and lease fencing do not prove physical callback stop or a diff --git a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json index 5135e0b4e..f643799d0 100644 --- a/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json +++ b/tests/Fixtures/contracts/activity-retry-policy/prepared-local.v2.json @@ -45,7 +45,8 @@ }, "scope": "existing v1 recovery/receipt source facts only; not a contract-2 writer or installed-role qualification", "prior_133_cases_mutated": false, - "extension_frozen": false + "extension_frozen": false, + "core_shape_review": "https://github.com/durable-workflow/server/pull/1034#issuecomment-6675154532" } }, "adoption_gates": [ @@ -140,7 +141,7 @@ "application failure retry selection" ], "recovery_sample": "for a fresh proven contract-2 recovery, the first locked engine observation that commits; never a claimed callback failure time or original lease-expiry timestamp", - "recovery_duplicate": "after scoped immutable identity/mode validation, return the matching committed recovery receipt and decision before any fresh clock, fence or policy calculation; rollback publishes none" + "recovery_duplicate": "after scoped immutable identity/mode, strict retained-snapshot and original Started-authority validation, return the matching committed recovery receipt and decision before fresh clock/fence/policy calculation. Later new-authoring configuration may differ; mutation of the canonical retained graph refuses and is never repaired from the receipt" }, "retry_disposition_proposal": { "boundary": "engine-owned object shared exactly by the stored outcome metadata and its prepared receipt; never accepted in a worker report", @@ -198,7 +199,8 @@ "unknown_members": "refused by a future consumer; never silently ignored", "conditional_common": { "timeout_kind": "required iff failure_source is engine_timeout; one of start_to_close, schedule_to_close, heartbeat; forbidden otherwise", - "engine_interruption": "only a fenced engine recovery branch; requested_delay_seconds and timeout_kind forbidden; any selected delay_source must be policy" + "engine_interruption": "only a fenced engine recovery branch; requested_delay_seconds and timeout_kind forbidden; any selected delay_source must be policy", + "engine_interruption_cap": "a validated snapshot-2 policy delay cannot exceed its normalized cap; retry_delay_exceeds_policy_cap is not a valid engine_interruption decision. A corrupt retained cap/delay relationship refuses before any decision, event, task, attempt or status mutation" }, "delay_source_rule": "failure_override only for a still-admitted application failure with a supplied valid override; otherwise policy. A preempted application override stays requested diagnostic input only.", "engine_timeout_terminal_classification": "reserved for overall schedule_to_close expiry; start_to_close/heartbeat before the total deadline may schedule the policy retry or terminate under their actual budget/cap/deadline branch" @@ -261,7 +263,7 @@ "cold_replay": "consume stored failed/timeout retry disposition and availability; never evaluate current policy/clock for that decision; no fetch-free completed replay claim", "timeout_scheduled": "an actual pre-total-deadline start_to_close/heartbeat timeout may record ActivityRetryScheduled with retry_reason timeout, actual timeout_kind and policy-selected delay; no worker override is applied", "interruption": "retain local_recovery and the engine RuntimeException provenance; scheduled uses the existing cold-replay retry reason; terminal uses ActivityFailed, except that an actually due timeout retains engine_timeout precedence", - "recovery_replay": "consume the stored recovery decision and original unknown stop state without current clock/policy/physical-state substitution; no callback is invoked by recovery", + "recovery_replay": "validate the exact original retained snapshot and Started authority, then consume the stored recovery decision and unknown stop state without recalculation from later new-authoring configuration, a fresh clock or physical-state substitution; do not repair a changed original graph and do not invoke a callback", "interruption_retry_reason": "cold_replay" }, "ranges": { @@ -2505,10 +2507,10 @@ } }, { - "id": "local_v2_recovery_terminal_cap", + "id": "local_v2_recovery_corrupt_snapshot_cap_refusal", "carrier": { "operation": "prepared_recovery", - "original_mode": "proven contract 2 and retained snapshot 2", + "original_mode": "original contract 2 and retained claim otherwise valid; the stored execution snapshot is deliberately corrupted", "previous_claim_loss": "independently proven under canonical locks", "current_recovery_claim": "independently scoped and valid", "engine_sample_seconds": 90, @@ -2517,24 +2519,22 @@ "attempt_number": 1, "max_attempts": 3, "policy_delay_seconds": 21, - "maximum_interval_seconds": 20 + "maximum_interval_seconds": 20, + "original_started_authority": "retains the original valid delay 5/cap 20 snapshot unchanged", + "retained_snapshot_mutation": "execution snapshot policy delay changed to 21 while cap remains 20" }, "expected": { - "failure_source": "engine_interruption", - "callback_stop_state": "unknown", - "requested_delay_member": "absent", - "timeout_kind_member": "absent", - "new_attempt_count": 0, - "callback_invocation_count": 0, - "decision": "terminal", - "terminal_classification": "retry_delay_exceeds_policy_cap", - "delay_source": "policy", - "candidate_delay_seconds": 21, - "retry_available_at_member": "absent", - "event_type": "ActivityFailed", + "verdict": "corrupt_retained_snapshot_refusal", + "retry_disposition_member": "absent", + "published_sample_count": 0, + "published_decision_count": 0, + "new_event_count": 0, "new_retry_task_count": 0, "new_retry_event_count": 0, - "maximum_interval_seconds": 20 + "new_attempt_count": 0, + "attempt_status_mutation_count": 0, + "durable_mutation_count": 0, + "callback_invocation_count": 0 } }, { @@ -2649,8 +2649,10 @@ "first_transaction": "committed", "original_reply": "lost", "identical_duplicate_sample_seconds": 110, - "current_policy": "changed", - "separate_stop_acknowledgement": "arrived after the committed recovery" + "separate_stop_acknowledgement": "arrived after the committed recovery", + "later_configuration_for_new_authoring": "changed independently of this original execution", + "duplicate_request_descriptor": "the identical original normalized descriptor", + "original_retained_snapshot_and_started_authority": "exact and unchanged" }, "expected": { "failure_source": "engine_interruption", @@ -2666,7 +2668,8 @@ "fresh_clock_sample_count": 0, "fresh_fence_calculation_count": 0, "new_retry_task_count": 0, - "new_retry_event_count": 0 + "new_retry_event_count": 0, + "original_retained_graph_mutation_count": 0 } }, { @@ -2949,6 +2952,29 @@ "new_retry_task_count": 1, "new_retry_event_count": 1 } + }, + { + "id": "local_v2_recovery_duplicate_canonical_snapshot_mutation", + "carrier": { + "first_recovery": "valid scoped contract-2 recovery committed with an exact retained snapshot and Started binding", + "matching_receipt": "the existing original committed receipt remains present", + "isolated_mutation": "canonical retained execution snapshot changed after commit; original Started authority and receipt remain unchanged", + "later_new_authoring_configuration": "irrelevant" + }, + "expected": { + "verdict": "original_preparation_authority_mismatch_refusal", + "retained_receipt_returned": false, + "new_retry_calculation_count": 0, + "new_published_sample_count": 0, + "new_published_decision_count": 0, + "new_event_count": 0, + "new_retry_task_count": 0, + "new_retry_event_count": 0, + "new_attempt_count": 0, + "attempt_status_mutation_count": 0, + "durable_mutation_count": 0, + "receipt_or_original_graph_repair_count": 0 + } } ], "prepared_policy_proposal": { @@ -3304,7 +3330,7 @@ "engine-observed interruption selected only when no activity timeout is due" ], "interruption_exception": "the existing engine-generated RuntimeException and message from recordInterruptedAttempt; never a worker-supplied exception, failure timestamp or timeout kind", - "policy": "retained snapshot 2 only, including non-retryable type, attempt budget, finite cap, checked availability and original deadline; no cap clamp, v1 fallback or batch-clock substitution", + "policy": "strictly validate the original retained snapshot 2 and Started binding before any decision; use that snapshot only, including non-retryable type, attempt budget, finite cap, checked availability and original deadline. Supplied policy delays already fit the cap; corrupt retained delay/cap relationships refuse without mutation, never a fabricated terminal-cap decision. No cap clamp, v1 fallback or batch-clock substitution", "worker_input": "no new worker-report member; engine_interruption is forbidden as report.outcome or report.failure_source, and engine output/requested-delay/timeout/stop metadata remains forbidden. A valid ordinary failed override remains an application report, never recovery provenance", "local_recovery": { "placement": "unchanged event payload.local_recovery, sibling of retry_disposition; never accepted from a worker",