From f5e19fbf2087dc626b970cf52e2c51da51614b23 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:19:07 -0600 Subject: [PATCH 1/5] feat: implement flat child-only all barriers --- rust/src/runtime/all_groups.rs | 22 +- rust/src/runtime/child_summary.rs | 39 +- rust/src/runtime/children.rs | 126 ++++- rust/src/runtime/continuations.rs | 60 ++- rust/src/runtime/selections.rs | 34 +- rust/tests/execution.rs | 2 + rust/tests/execution/parallel_children.rs | 558 ++++++++++++++++++++++ 7 files changed, 784 insertions(+), 57 deletions(-) create mode 100644 rust/tests/execution/parallel_children.rs diff --git a/rust/src/runtime/all_groups.rs b/rust/src/runtime/all_groups.rs index c7cc4834e..8f8236b43 100644 --- a/rust/src/runtime/all_groups.rs +++ b/rust/src/runtime/all_groups.rs @@ -1,4 +1,4 @@ -//! Published activity/timer all barriers retain every authored ancestor. +//! Published all barriers retain every qualified authored ancestor. use super::{Result, refuse}; use axum::http::StatusCode; use serde_json::{Value, json}; @@ -38,6 +38,7 @@ fn entry(value: &Value, leaf: &str) -> Result { let prefix = match kind { "activity" => "parallel-activities", "timer" => "parallel-timers", + "child" => "parallel-children", _ => "parallel-calls", }; if value["parallel_group_id"] != format!("{prefix}:{base}:{size}") { @@ -50,7 +51,7 @@ fn entry(value: &Value, leaf: &str) -> Result { } pub(super) fn metadata(value: &Value, kind: &str) -> Result { - if !matches!(kind, "activity" | "timer") { + if !matches!(kind, "activity" | "timer" | "child") { return Err(invalid()); } let top = entry(value, kind)?; @@ -58,6 +59,11 @@ pub(super) fn metadata(value: &Value, kind: &str) -> Result { .as_array() .filter(|path| !path.is_empty()) .ok_or_else(invalid)?; + // Child selections, nested barriers and mixed child groups have not been + // qualified. Keep their refusal independent of activity/timer paths. + if kind == "child" && (path.len() != 1 || top["parallel_group_kind"] != "child") { + return Err(invalid()); + } let path = path .iter() .map(|value| entry(value, kind)) @@ -95,6 +101,7 @@ pub(super) fn validate_batch(commands: &[Value]) -> Result<()> { let kind = match command["type"].as_str() { Some("schedule_activity") => "activity", Some("start_timer") => "timer", + Some("start_child_workflow") => "child", _ => return Err(invalid()), }; let metadata = metadata(command, kind)?; @@ -156,6 +163,8 @@ impl AllHistory { ("activity", "activity_execution_id") } else if event.starts_with("Timer") { ("timer", "timer_id") + } else if event.starts_with("Child") { + ("child", "child_call_id") } else { return true; }; @@ -196,7 +205,10 @@ impl AllHistory { { return false; } - if matches!(event, "ActivityScheduled" | "TimerScheduled") { + if matches!( + event, + "ActivityScheduled" | "TimerScheduled" | "ChildWorkflowScheduled" + ) { let group = self.groups.entry(group_id).or_insert_with(|| Group { size, members: BTreeMap::new(), @@ -243,6 +255,10 @@ impl AllHistory { | "ActivityCancelled" | "TimerFired" | "TimerCancelled" + | "ChildRunCompleted" + | "ChildRunFailed" + | "ChildRunCancelled" + | "ChildRunTerminated" ) { member.terminal = true; if group.members.len() == size && group.members.values().all(|member| member.terminal) { diff --git a/rust/src/runtime/child_summary.rs b/rust/src/runtime/child_summary.rs index 3421a22af..e347ae75a 100644 --- a/rust/src/runtime/child_summary.rs +++ b/rust/src/runtime/child_summary.rs @@ -18,6 +18,7 @@ pub(super) struct ChildSummaryHistory { struct ExpectedChild { id: String, + parallel: Value, /// The first run, which the parent's wait keeps naming. run: String, instance: String, @@ -58,7 +59,12 @@ impl ChildSummaryHistory { if !previous.same_call(payload) || previous.status != "waiting" { return false; } - let text = |key: &str| payload[key].as_str().filter(|v| !v.is_empty()).map(str::to_owned); + let text = |key: &str| { + payload[key] + .as_str() + .filter(|v| !v.is_empty()) + .map(str::to_owned) + }; match event { "ChildRunStarted" if !previous.started => { if payload["child_run_number"] != 1 @@ -69,10 +75,15 @@ impl ChildSummaryHistory { } previous.started = true; } - "ChildRunStarted" if payload["retry_attempt"].is_i64() => { - let (Some(run), Some(link), Some(number)) = - (text("child_workflow_run_id"), text("workflow_link_id"), payload["child_run_number"].as_i64()) - else { + "ChildRunStarted" + if payload["retry_attempt"].is_i64() + || previous.parallel.get("parallel_group_id").is_some() => + { + let (Some(run), Some(link), Some(number)) = ( + text("child_workflow_run_id"), + text("workflow_link_id"), + payload["child_run_number"].as_i64(), + ) else { return false; }; if number <= previous.run_number || run == previous.latest_run || link == previous.latest_link { @@ -116,11 +127,9 @@ impl ExpectedChild { }; let id = text("child_call_id")?; let label = text("child_workflow_type")?; - if payload["workflow_link_id"] != id - || payload["child_workflow_class"] != label - || !payload["parallel_group_id"].is_null() - || !payload["parallel_group_path"].is_null() - { + let parallel = super::selections::metadata(payload, "child").ok()?; + super::selections::check_sequence(¶llel, payload["sequence"].as_i64()?).ok()?; + if payload["workflow_link_id"] != id || payload["child_workflow_class"] != label { return None; } Some(Self { @@ -128,6 +137,7 @@ impl ExpectedChild { run: text("child_workflow_run_id")?, instance: text("child_workflow_instance_id")?, label, + parallel, started: false, status: "waiting", latest_run: text("child_workflow_run_id")?, @@ -142,8 +152,7 @@ impl ExpectedChild { && payload["child_workflow_instance_id"] == self.instance.as_str() && payload["child_workflow_type"] == self.label.as_str() && payload["child_workflow_class"] == self.label.as_str() - && payload["parallel_group_id"].is_null() - && payload["parallel_group_path"].is_null() + && super::selections::metadata(payload, "child").ok().as_ref() == Some(&self.parallel) } fn wait_status(&self) -> &'static str { @@ -173,7 +182,7 @@ where history: &ChildSummaryHistory, summary: &mut ActiveSummary, ) -> Result { - let rows = Self::query("SELECT w.*,c.status AS call_status,c.resolved_child_run_id,c.resolved_child_instance_id,c.child_workflow_type,c.metadata AS call_metadata,l.id AS link_id,l.child_workflow_run_id AS link_run,l.child_workflow_instance_id AS link_instance FROM workflow_run_waits w LEFT JOIN workflow_child_calls c ON c.parent_workflow_run_id=w.workflow_run_id AND c.sequence=w.sequence LEFT JOIN workflow_links l ON l.parent_workflow_run_id=w.workflow_run_id AND l.sequence=w.sequence AND l.link_type='child_workflow' AND l.child_workflow_run_id=c.resolved_child_run_id WHERE w.workflow_run_id=$1 AND w.kind='child' ORDER BY w.sequence") + let rows = Self::query("SELECT w.*,c.status AS call_status,c.resolved_child_run_id,c.resolved_child_instance_id,c.child_workflow_type,c.metadata AS call_metadata,l.id AS link_id,l.child_workflow_run_id AS link_run,l.child_workflow_instance_id AS link_instance,l.parallel_group_path AS link_group_path FROM workflow_run_waits w LEFT JOIN workflow_child_calls c ON c.parent_workflow_run_id=w.workflow_run_id AND c.sequence=w.sequence LEFT JOIN workflow_links l ON l.parent_workflow_run_id=w.workflow_run_id AND l.sequence=w.sequence AND l.link_type='child_workflow' AND l.child_workflow_run_id=c.resolved_child_run_id WHERE w.workflow_run_id=$1 AND w.kind='child' ORDER BY w.sequence") .bind(run_id).fetch_all(&mut **tx).await?; if rows.len() != history.calls.len() { return Ok(false); @@ -307,6 +316,8 @@ where .is_some_and(|v| v["child_call_id"] == child.id) && matches("link_id", &child.latest_link)? && matches("link_run", &child.latest_run)? - && matches("link_instance", &child.instance)?) + && matches("link_instance", &child.instance)? + && DB::optional_document_row(row, "link_group_path")? + == child.parallel.get("parallel_group_path").cloned()) } } diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index b38184cd4..7db68ba6c 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -1,14 +1,13 @@ //! Sequential child outcomes and abandoned-parent cleanup retain the original call. //! A retried child keeps the call under a new link (`child_retries`). -//! Parallel child groups remain unqualified and are refused at admission -//! rather than silently discarded. +//! Flat child-only all groups keep their path on every physical child link. use super::schedule_storage::iso8601; use super::transaction::Transaction; use super::{ Result, backend::Backend, envelope, refuse, - store::{Store, id, now, reject_fields}, + store::{Store, id, now}, text, wire, }; use axum::http::StatusCode; @@ -44,7 +43,7 @@ enum ChildParentResume { } pub(super) fn validate_child(command: &Value) -> Result<()> { - reject_fields( + super::selections::fields( command, &[ "type", @@ -59,6 +58,15 @@ pub(super) fn validate_child(command: &Value) -> Result<()> { "await", ], )?; + let parallel = super::selections::metadata(command, "child")?; + // This slice qualifies result barriers. A committed-start handle inside + // a barrier has a different wakeup contract and stays separately refused. + if parallel.get("parallel_group_id").is_some() && command["await"] == AWAIT_STARTED { + return Err(refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "development_parallel_group_unqualified", + )); + } if text(command, "workflow_type")?.len() > 191 || command.get("queue").is_some_and(|queue| { queue @@ -80,19 +88,35 @@ pub(super) fn validate_child(command: &Value) -> Result<()> { if let Some(seconds) = super::child_retries::timeout(command, field) && super::child_retries::after(now, seconds).is_none() { - return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_child_timeout")); + return Err(refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_child_timeout", + )); } } // PHP reads a backoff object's values as a list (`array_values`). - if let Some(mut backoff) = super::child_retries::list(&command["retry_policy"]["backoff_seconds"]) - && backoff.any(|entry| entry.as_i64().is_some_and(|seconds| super::child_retries::after(now, seconds).is_none())) + if let Some(mut backoff) = + super::child_retries::list(&command["retry_policy"]["backoff_seconds"]) + && backoff.any(|entry| { + entry + .as_i64() + .is_some_and(|seconds| super::child_retries::after(now, seconds).is_none()) + }) { - return Err(refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_child_retry_policy")); + return Err(refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_child_retry_policy", + )); } Ok(()) } -const PARENT_CLOSE_POLICIES: [&str; 4] = ["abandon", "request_cancel", "request_cancellation", "terminate"]; +const PARENT_CLOSE_POLICIES: [&str; 4] = [ + "abandon", + "request_cancel", + "request_cancellation", + "terminate", +]; const CANCELLATION_POLICIES: [&str; 3] = ["try_cancel", "wait_cancellation_completed", "abandon"]; /// A child's parent-close and cancellation policies. @@ -120,21 +144,34 @@ pub(super) fn policies(command: &Value) -> Result { None | Some(Value::Null) => None, Some(Value::String(text)) => Some(text.trim().to_owned()).filter(|text| !text.is_empty()), Some(_) => { - errors.push((format!("commands.0.{field}"), vec![format!("The commands.0.{field} field must be a string.")])); + errors.push(( + format!("commands.0.{field}"), + vec![format!("The commands.0.{field} field must be a string.")], + )); None } }; let parent_close = read("parent_close_policy"); let cancellation = read("cancellation_policy"); - if cancellation.as_deref().is_some_and(|value| !CANCELLATION_POLICIES.contains(&value)) { - errors.push(("commands.0.cancellation_policy".into(), vec!["The selected commands.0.cancellation_policy is invalid.".into()])); + if cancellation + .as_deref() + .is_some_and(|value| !CANCELLATION_POLICIES.contains(&value)) + { + errors.push(( + "commands.0.cancellation_policy".into(), + vec!["The selected commands.0.cancellation_policy is invalid.".into()], + )); } if let Some(error) = super::laravel_validation::failure_messages(errors) { return Err(error); } if parent_close.as_deref().is_some_and(|value| !PARENT_CLOSE_POLICIES.contains(&value)) { let message = "The parent_close_policy must be one of: abandon, request_cancel, request_cancellation, terminate."; - return Err(super::laravel_validation::failure(vec![("commands.0.parent_close_policy", vec![message])]).expect("one message")); + return Err(super::laravel_validation::failure(vec![( + "commands.0.parent_close_policy", + vec![message], + )]) + .expect("one message")); } Ok(ChildPolicies { parent_close: parent_close.unwrap_or_else(|| "abandon".into()), @@ -324,6 +361,8 @@ where text(command, "queue").map(str::to_owned) })?; let arguments = envelope(command, "arguments")?; + let parallel = super::selections::metadata(command, "child")?; + super::selections::check_sequence(¶llel, sequence)?; let policies = policies(command)?; let child_instance = id(); let child_run = id(); @@ -352,14 +391,24 @@ where // `inheritTypedVisibilityMetadata`: the child takes its parent's memos // and search attributes. let memos = Self::inheritable_memos(tx, &parent_run).await?; - Self::inherit_visibility(tx, memos, &parent_run, &child_run, &child_instance, started_at).await?; + Self::inherit_visibility( + tx, + memos, + &parent_run, + &child_run, + &child_instance, + started_at, + ) + .await?; Self::query("INSERT INTO workflow_child_calls(parent_workflow_run_id,parent_workflow_instance_id,sequence,child_workflow_type,child_workflow_class,resolved_child_instance_id,resolved_child_run_id,parent_close_policy,cancellation_propagation,queue,status,scheduled_at,started_at,arguments,metadata,created_at,updated_at,retry_policy,timeout_policy) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,($9<>'abandon'),$10,'started',$11,$12,$13,$14,$15,$16,$17,$18)") .bind(&parent_run).bind(&parent_instance).bind(sequence).bind(workflow_type).bind(workflow_type).bind(&child_instance).bind(&child_run).bind(&policies.parent_close).bind(&policies.cancellation).bind(&queue).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)) .bind(DB::document(&json!({"payload":arguments}))).bind(DB::document(&json!({"child_call_id":call,"attempt_count":1,"cancellation_policy":policies.cancellation}))).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)) .bind(retry_policy.as_ref().map(DB::document)).bind(timeout_policy.as_ref().map(DB::document)).execute(&mut **tx).await?; - Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at) VALUES ($1,'child_workflow',$2,$3,$4,$5,$6,TRUE,$7,$8,$9)") - .bind(&call).bind(sequence).bind(&parent_instance).bind(&parent_run).bind(&child_instance).bind(&child_run).bind(&policies.parent_close).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)).execute(&mut **tx).await?; - let scheduled = json!({"sequence":sequence,"workflow_link_id":call,"child_call_id":call,"child_workflow_instance_id":child_instance,"child_workflow_run_id":child_run,"child_workflow_class":workflow_type,"child_workflow_type":workflow_type,"parent_close_policy":policies.parent_close,"cancellation_policy":policies.cancellation,"retry_policy":retry_policy,"timeout_policy":timeout_policy}); + Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at,parallel_group_path) VALUES ($1,'child_workflow',$2,$3,$4,$5,$6,TRUE,$7,$8,$9,$10)") + .bind(&call).bind(sequence).bind(&parent_instance).bind(&parent_run).bind(&child_instance).bind(&child_run).bind(&policies.parent_close).bind(DB::bind_time(started_at)).bind(DB::bind_time(started_at)) + .bind(parallel.get("parallel_group_path").map(DB::document)).execute(&mut **tx).await?; + let mut scheduled = json!({"sequence":sequence,"workflow_link_id":call,"child_call_id":call,"child_workflow_instance_id":child_instance,"child_workflow_run_id":child_run,"child_workflow_class":workflow_type,"child_workflow_type":workflow_type,"parent_close_policy":policies.parent_close,"cancellation_policy":policies.cancellation,"retry_policy":retry_policy,"timeout_policy":timeout_policy}); + super::selections::merge(&mut scheduled, ¶llel); Self::append( tx, &parent_run, @@ -400,7 +449,10 @@ where let turn = json!({"workflow_wait_kind":"child_started","child_call_id":call,"child_workflow_run_id":child_run, "resume_source_kind":"child_workflow_run","resume_source_id":child_run,"workflow_sequence":sequence, "workflow_event_type":"ChildRunStarted"}); - Ok(StartedChild { task: child_task, turn }) + Ok(StartedChild { + task: child_task, + turn, + }) } /// The parent's start turn (Server #966): one fresh workflow task, created @@ -486,6 +538,10 @@ where if !matches!( DB::string(&parent, "status")?.as_str(), "pending" | "running" | "waiting" + let parallel = super::selections::child_metadata(DB::optional_document_row( + link, + "parallel_group_path", + )?)?; ) || DB::string(&parent, "current_run_id")? != parent_run { return Ok(()); @@ -560,7 +616,6 @@ where .execute(&mut **tx) .await?; } - let parent_resume = Self::child_parent_resume(tx, &parent).await?; // The event names the child run's closed reason: a run that timed // out failed with `timed_out`. The call keeps its status. let closed_reason = match resolution { @@ -613,7 +668,17 @@ where /// the same call. Returns whether a retry started. async fn start_child_retry( tx: &mut Transaction<'_, DB>, + super::selections::merge(&mut payload, ¶llel); parent: &DB::Row, + // Record the outcome even while the all barrier remains closed. The + // last successful member observes every committed call under the same + // parent transition lock; any failed member wakes the parent at once. + if matches!(resolution, ChildResolution::Completed(_)) + && !Self::all_groups_closed(tx, &parent_run, ¶llel).await? + { + return Ok(()); + } + let parent_resume = Self::child_parent_resume(tx, &parent).await?; link: &DB::Row, call: &DB::Row, failed: &DB::Row, @@ -641,6 +706,10 @@ where .filter(|policy| policy.is_object() || policy.is_array()) .unwrap_or(Value::Null) }; + let parallel = super::selections::child_metadata(DB::optional_document_row( + link, + "parallel_group_path", + )?)?; if policy.is_null() { return Ok(false); } @@ -652,7 +721,10 @@ where let failure = Self::query("SELECT CASE WHEN non_retryable THEN 1 ELSE 0 END AS non_retryable_flag,exception_class FROM workflow_failures WHERE workflow_run_id=$1 ORDER BY created_at,id LIMIT 1") .bind(&failed_run).fetch_optional(&mut **tx).await?; let failure = match failure { - Some(row) => Some((DB::number(&row, "non_retryable_flag")? != 0, DB::optional_string(&row, "exception_class")?)), + Some(row) => Some(( + DB::number(&row, "non_retryable_flag")? != 0, + DB::optional_string(&row, "exception_class")?, + )), None => None, }; let failed_event = Self::query("SELECT payload FROM workflow_history_events WHERE workflow_run_id=$1 AND event_type='WorkflowFailed' ORDER BY sequence DESC LIMIT 1") @@ -669,8 +741,12 @@ where let run_timeout = timeout_policy["run_timeout_seconds"].as_i64(); let at = now(); let backoff = super::child_retries::backoff_seconds(&policy, attempts); - let available_at = super::child_retries::after(at, backoff) - .ok_or_else(|| refuse(StatusCode::UNPROCESSABLE_ENTITY, "invalid_child_retry_policy"))?; + let available_at = super::child_retries::after(at, backoff).ok_or_else(|| { + refuse( + StatusCode::UNPROCESSABLE_ENTITY, + "invalid_child_retry_policy", + ) + })?; let instance = DB::string(failed, "workflow_instance_id")?; // The latest run as a row: an aggregate of MySQL's unsigned column // does not decode as a signed integer. @@ -696,9 +772,10 @@ where .bind(&retry_run).bind(run_number).bind(run_number).bind(DB::bind_time(at)).bind(&instance).execute(&mut **tx).await?; let parent_close = DB::optional_string(link, "parent_close_policy")?.unwrap_or_else(|| "abandon".into()); let retry_link = id(); - Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at) VALUES ($1,'child_workflow',$2,$3,$4,$5,$6,TRUE,$7,$8,$9)") + Self::query("INSERT INTO workflow_links(id,link_type,sequence,parent_workflow_instance_id,parent_workflow_run_id,child_workflow_instance_id,child_workflow_run_id,is_primary_parent,parent_close_policy,created_at,updated_at,parallel_group_path) VALUES ($1,'child_workflow',$2,$3,$4,$5,$6,TRUE,$7,$8,$9,$10)") .bind(&retry_link).bind(sequence).bind(DB::string(parent, "workflow_instance_id")?).bind(&parent_run).bind(&instance).bind(&retry_run) - .bind(&parent_close).bind(DB::bind_time(at)).bind(DB::bind_time(at)).execute(&mut **tx).await?; + .bind(&parent_close).bind(DB::bind_time(at)).bind(DB::bind_time(at)) + .bind(parallel.get("parallel_group_path").map(DB::document)).execute(&mut **tx).await?; let workflow_type = DB::string(failed, "workflow_type")?; let workflow_class = DB::string(failed, "workflow_class")?; let deadlines = json!({"execution_timeout_seconds":execution_timeout,"run_timeout_seconds":run_timeout, @@ -743,6 +820,7 @@ where metadata["last_retry_backoff_seconds"] = json!(backoff); Self::query("UPDATE workflow_child_calls SET resolved_child_instance_id=$1,resolved_child_run_id=$2,status='started',started_at=$3,metadata=$4,updated_at=$5 WHERE id=$6") .bind(&instance).bind(&retry_run).bind(DB::bind_time(at)).bind(DB::document(&metadata)).bind(DB::bind_time(at)) + super::selections::merge(&mut started, ¶llel); .bind(DB::number(call, "id")?).execute(&mut **tx).await?; tx.summary_changed(&parent_run); Ok(true) diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index e35690e12..af864e175 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -91,10 +91,19 @@ where /// The memos `MemoUpsertService::inheritFromParent` copies from a run: /// each key and value, with its portable value when that is current. - pub(super) async fn inheritable_memos(tx: &mut Transaction<'_, DB>, from: &str) -> Result> { - let rows = Self::query(DB::MEMO_READ).bind(from).fetch_all(&mut **tx).await?; + pub(super) async fn inheritable_memos( + tx: &mut Transaction<'_, DB>, + from: &str, + ) -> Result> { + let rows = Self::query(DB::MEMO_READ) + .bind(from) + .fetch_all(&mut **tx) + .await?; if rows.len() > super::memos::MAX_ENTRIES { - return Err(refuse(StatusCode::CONFLICT, "workflow_memo_validation_failed")); + return Err(refuse( + StatusCode::CONFLICT, + "workflow_memo_validation_failed", + )); } let mut memos = Vec::with_capacity(rows.len()); for memo in rows { @@ -106,7 +115,11 @@ where } else { None }; - memos.push((DB::string(&memo, "key")?, DB::document_row(&memo, "value")?, portable)); + memos.push(( + DB::string(&memo, "key")?, + DB::document_row(&memo, "value")?, + portable, + )); } Ok(memos) } @@ -259,14 +272,22 @@ where .map(|row| DB::document_row(&row, "payload")) .transpose()? .unwrap_or(Value::Null); - let call_id = original["child_call_id"].as_str().filter(|id| !id.is_empty()).map(str::to_owned); - let reference = match original["parent_workflow_run_id"].as_str().filter(|id| !id.is_empty()) { - None => json!({"child_call_id":call_id,"parent_workflow_instance_id":null,"parent_workflow_run_id":null,"parent_sequence":null}), + let call_id = original["child_call_id"] + .as_str() + .filter(|id| !id.is_empty()) + .map(str::to_owned); + let reference = match original["parent_workflow_run_id"] + .as_str() + .filter(|id| !id.is_empty()) + { + None => { + json!({"child_call_id":call_id,"parent_workflow_instance_id":null,"parent_workflow_run_id":null,"parent_sequence":null}) + } Some(parent_run) => json!({"child_call_id":call_id, "parent_workflow_instance_id":original["parent_workflow_instance_id"].as_str().unwrap_or(&original["workflow_instance_id"].as_str().unwrap_or_default()), "parent_workflow_run_id":parent_run,"parent_sequence":original["parent_sequence"].as_i64()}), }; - let links = Self::query("SELECT id,sequence,parent_workflow_run_id,child_workflow_instance_id,parent_close_policy FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow' ORDER BY id") + let links = Self::query("SELECT id,sequence,parent_workflow_run_id,child_workflow_instance_id,parent_close_policy,parallel_group_path FROM workflow_links WHERE child_workflow_run_id=$1 AND link_type='child_workflow' ORDER BY id") .bind(old).fetch_all(&mut **tx).await?; for parent_link in links { let link = id(); @@ -335,6 +356,11 @@ where .map(|names| names.iter().filter_map(Value::as_str).collect()) .unwrap_or_default(); let mut messages = Vec::new(); + let parallel = super::selections::child_metadata(DB::optional_document_row( + &parent_link, + "parallel_group_path", + )?)?; + super::selections::merge(&mut payload, ¶llel); for row in Self::query("SELECT * FROM workflow_signal_records WHERE workflow_run_id=$1 AND target_scope='instance' AND status='received' AND closed_at IS NULL") .bind(old).fetch_all(&mut **tx).await? { @@ -421,8 +447,12 @@ where .bind(next).bind(next).bind(&update_id).execute(&mut **tx).await?; let command_id = DB::optional_string(update, "workflow_command_id")?; let command = match &command_id { - Some(command_id) => Self::query("SELECT command_type,status FROM workflow_commands WHERE id=$1") - .bind(command_id).fetch_optional(&mut **tx).await?, + Some(command_id) => { + Self::query("SELECT command_type,status FROM workflow_commands WHERE id=$1") + .bind(command_id) + .fetch_optional(&mut **tx) + .await? + } None => None, }; let command_id = match command { @@ -442,6 +472,14 @@ where "workflow_instance_id":workflow,"workflow_run_id":next, "update_name":DB::string(update, "update_name")?, "arguments":DB::optional_string(update, "arguments")?.as_deref().map(wire)}); - Self::append(tx, next, "UpdateAccepted", payload, None, command_id.as_deref()).await + Self::append( + tx, + next, + "UpdateAccepted", + payload, + None, + command_id.as_deref(), + ) + .await } } diff --git a/rust/src/runtime/selections.rs b/rust/src/runtime/selections.rs index 9c4cb5d58..5e6faf211 100644 --- a/rust/src/runtime/selections.rs +++ b/rust/src/runtime/selections.rs @@ -104,6 +104,7 @@ pub(super) fn validate_batch(commands: &[Value]) -> Result<()> { let kind = match command["type"].as_str() { Some("schedule_activity") => "activity", Some("start_timer") => "timer", + Some("start_child_workflow") => "child", _ => return Err(unqualified()), }; metadata(command, kind)?; @@ -174,6 +175,14 @@ pub(super) fn merge(payload: &mut Value, metadata: &Value) { } pub(super) fn activity_metadata(path: Option) -> Result { + path_metadata(path, "activity") +} + +pub(super) fn child_metadata(path: Option) -> Result { + path_metadata(path, "child") +} + +fn path_metadata(path: Option, kind: &str) -> Result { let Some(path) = path else { return Ok(json!({})); }; @@ -184,7 +193,7 @@ pub(super) fn activity_metadata(path: Option) -> Result { .ok_or_else(unqualified)?; let mut value = leaf.clone(); value["parallel_group_path"] = path.clone(); - metadata(&value, "activity") + metadata(&value, kind) } impl Store @@ -271,8 +280,13 @@ where } /// `groupCompletedSuccessfully` for each all-of group on a member's path: - /// every member sequence has a fired timer or a completed activity. - async fn all_groups_closed(tx: &mut Transaction<'_, DB>, run_id: &str, metadata: &Value) -> Result { + /// Every member sequence has a fired timer, completed activity or completed + /// child call. A retried/continued child is still one original call. + pub(super) async fn all_groups_closed( + tx: &mut Transaction<'_, DB>, + run_id: &str, + metadata: &Value, + ) -> Result { let Some(path) = metadata["parallel_group_path"].as_array() else { return Ok(true); }; @@ -286,8 +300,18 @@ where let last = base + size - 1; let mut closed = std::collections::BTreeSet::new(); for (sql, done) in [ - ("SELECT sequence,status FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence>=$2 AND sequence<=$3", "fired"), - ("SELECT sequence,status FROM activity_executions WHERE workflow_run_id=$1 AND sequence>=$2 AND sequence<=$3", "completed"), + ( + "SELECT sequence,status FROM workflow_run_timers WHERE workflow_run_id=$1 AND sequence>=$2 AND sequence<=$3", + "fired", + ), + ( + "SELECT sequence,status FROM activity_executions WHERE workflow_run_id=$1 AND sequence>=$2 AND sequence<=$3", + "completed", + ), + ( + "SELECT sequence,status FROM workflow_child_calls WHERE parent_workflow_run_id=$1 AND sequence>=$2 AND sequence<=$3", + "completed", + ), ] { for row in Self::query(sql).bind(run_id).bind(base).bind(last).fetch_all(&mut **tx).await? { if DB::string(&row, "status")? == done { diff --git a/rust/tests/execution.rs b/rust/tests/execution.rs index 5fe3195d4..3aa69c64a 100644 --- a/rust/tests/execution.rs +++ b/rust/tests/execution.rs @@ -287,6 +287,8 @@ mod timed_condition_waits; #[path = "execution/parallel_all.rs"] mod parallel_all; +#[path = "execution/parallel_children.rs"] +mod parallel_children; #[path = "execution/workflow_failures.rs"] mod workflow_failures; diff --git a/rust/tests/execution/parallel_children.rs b/rust/tests/execution/parallel_children.rs new file mode 100644 index 000000000..5bcc9d9a9 --- /dev/null +++ b/rust/tests/execution/parallel_children.rs @@ -0,0 +1,558 @@ +use super::*; + +fn group(size: usize, index: usize) -> Value { + json!({"parallel_group_id":format!("parallel-children:1:{size}"),"parallel_group_kind":"child", + "parallel_group_base_sequence":1,"parallel_group_size":size,"parallel_group_index":index}) +} + +fn children(size: usize) -> Value { + json!( + (0..size) + .map(|index| { + let mut command = child_command(9007199254740993 + index as i64); + command["queue"] = json!("children"); + command + .as_object_mut() + .unwrap() + .extend(group(size, index).as_object().unwrap().clone()); + command["parallel_group_path"] = json!([group(size, index)]); + command + }) + .collect::>() + ) +} + +async fn workers(app: &Router) { + register(app, "parent-worker", json!(["echo"]), json!([])).await; + let (status, body) = request(app, "POST", "/api/worker/register", json!({"worker_id":"child-worker", + "task_queue":"children","runtime":"php","supported_workflow_types":["child"],"supported_activity_types":[]})).await; + assert_eq!(status, StatusCode::CREATED, "{body}"); +} + +async fn child_poll(app: &Router) -> Value { + let (status, body) = request( + app, + "POST", + "/api/worker/workflow-tasks/poll", + json!({ + "worker_id":"child-worker","task_queue":"children","timeout_seconds":0}), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + body["task"].clone() +} + +fn done(value: i64) -> Value { + json!([{"type":"complete_workflow","result":envelope(Payload::Long(value))}]) +} + +async fn count(database: &TestDatabase, sql: String) -> i64 { + task_retry::rows(database, Box::leak(sql.into_boxed_str())).await +} + +async fn parent_tasks(database: &TestDatabase, run: &Value, status: &str) -> i64 { + count(database, format!("SELECT COUNT(*) FROM workflow_tasks WHERE workflow_run_id='{}' AND task_type='workflow' AND status='{status}'", run["run_id"].as_str().unwrap())).await +} + +async fn observation(app: &Router, run: &Value) -> Value { + let (status, value) = request( + app, + "GET", + &format!( + "/api/workflows/{}/runs/{}/observation", + run["workflow_id"].as_str().unwrap(), + run["run_id"].as_str().unwrap() + ), + Value::Null, + ) + .await; + assert_eq!(status, StatusCode::OK, "{value}"); + value +} + +async fn observed(app: &Router, run: &Value, open: usize) { + let value = observation(app, run).await; + assert_eq!(value["summary_state"], "observed", "{value}"); + assert_eq!( + value["current_waits"]["returned_count"], + open.min(50), + "{value}" + ); +} + +fn scheduled(history: &Value) -> Vec<&Value> { + history + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == "ChildWorkflowScheduled") + .map(|event| &event["payload"]) + .collect() +} + +fn check_paths(history: &Value, commands: &Value) { + for event in history + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"].as_str().unwrap().starts_with("Child")) + { + let payload = &event["payload"]; + let index = payload["sequence"].as_u64().unwrap() as usize - 1; + assert_eq!( + payload["parallel_group_path"], commands[index]["parallel_group_path"], + "{event}" + ); + assert_eq!( + payload["parallel_group_id"], + commands[index]["parallel_group_id"] + ); + } +} + +#[tokio::test] +async fn flat_children_keep_original_ids_out_of_order_results_and_one_cold_barrier_wakeup() { + let database = TestDatabase::new().await; + let first = database.open().await.unwrap(); + let app = router(first.clone()); + workers(&app).await; + let run = start(&app, "child-all-cold").await; + let initial = poll(&app, "parent-worker", "workflow").await; + let commands = children(3); + let admitted = finish_task(&app, &initial, commands.clone()).await; + assert_eq!(admitted.0, StatusCode::OK, "{}", admitted.1); + assert_eq!(admitted.1["created_task_ids"].as_array().unwrap().len(), 3); + let prefix = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + assert_eq!(prefix.as_array().unwrap().len(), 8); + check_paths(&prefix, &commands); + observed(&app, &run, 3).await; + assert_eq!( + finish_task(&app, &initial, commands.clone()).await.1["recorded"], + false + ); + assert_eq!( + run_history(&app, &run["workflow_id"], &run["run_id"]).await, + prefix + ); + drop(app); + first.close().await; + + let (one, two) = tokio::join!(database.open(), database.open()); + let (one, two) = (one.unwrap(), two.unwrap()); + let (app_a, app_b) = (router(one.clone()), router(two.clone())); + let mut tasks = Vec::new(); + for _ in 0..3 { + tasks.push(child_poll(&app_a).await); + } + tasks.sort_by_key(|task| { + scheduled(&prefix) + .iter() + .position(|payload| payload["child_workflow_run_id"] == task["run_id"]) + .unwrap() + }); + for (index, task) in tasks.iter().enumerate() { + assert_eq!( + task["workflow_id"], + scheduled(&prefix)[index]["child_workflow_instance_id"] + ); + assert_eq!(task["arguments"], commands[index]["arguments"]); + } + assert_eq!( + finish_task(&app_b, &tasks[2], done(9007199254740995)) + .await + .0, + StatusCode::OK + ); + assert!(poll(&app_a, "parent-worker", "workflow").await.is_null()); + assert_eq!(parent_tasks(&database, &run, "ready").await, 0); + observed(&app_a, &run, 2).await; + // Competing nodes close the remaining members. The parent transition lock + // must make exactly one ready resume observe the complete barrier. + let (a, b) = tokio::join!( + finish_task(&app_a, &tasks[0], done(9007199254740993)), + finish_task(&app_b, &tasks[1], done(9007199254740994)) + ); + assert_eq!(a.0, StatusCode::OK, "{}", a.1); + assert_eq!(b.0, StatusCode::OK, "{}", b.1); + assert_eq!(parent_tasks(&database, &run, "ready").await, 1); + let history = run_history(&app_a, &run["workflow_id"], &run["run_id"]).await; + assert_eq!( + &history.as_array().unwrap()[..8], + prefix.as_array().unwrap() + ); + check_paths(&history, &commands); + for event in history + .as_array() + .unwrap() + .iter() + .filter(|event| event["event_type"] == "ChildRunCompleted") + { + let index = event["payload"]["sequence"].as_u64().unwrap() as usize - 1; + let original = scheduled(&prefix)[index]; + for key in [ + "child_call_id", + "workflow_link_id", + "child_workflow_instance_id", + "child_workflow_run_id", + ] { + assert_eq!(event["payload"][key], original[key]); + } + assert_eq!( + event["payload"]["result"], + done(9007199254740993 + index as i64)[0]["result"] + ); + } + assert_eq!( + finish_task(&app_a, &initial, commands).await.1["recorded"], + false + ); + assert_eq!( + finish_task(&app_a, &tasks[1], done(9007199254740994)) + .await + .1["recorded"], + false + ); + assert_eq!( + run_history(&app_a, &run["workflow_id"], &run["run_id"]).await, + history + ); + observed(&app_a, &run, 0).await; + let resumed = poll(&app_a, "parent-worker", "workflow").await; + assert_eq!(resumed["run_id"], run["run_id"]); + let result = envelope(Payload::Array( + (0..3) + .map(|index| Payload::Long(9007199254740993 + index)) + .collect(), + )); + assert_eq!( + finish_task( + &app_b, + &resumed, + json!([{"type":"complete_workflow","result":result}]) + ) + .await + .0, + StatusCode::OK + ); + assert!(poll(&app_a, "parent-worker", "workflow").await.is_null()); + let final_history = run_history(&app_a, &run["workflow_id"], &run["run_id"]).await; + assert_eq!( + final_history.as_array().unwrap().last().unwrap()["payload"]["output"], + result + ); + one.close().await; + two.close().await; + database.remove().await; +} + +#[tokio::test] +async fn failed_cancelled_and_terminated_members_wake_immediately_without_reopening_on_success() { + for close in ["fail", "cancel", "terminate"] { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + workers(&app).await; + let run = start(&app, &format!("child-all-{close}")).await; + let initial = poll(&app, "parent-worker", "workflow").await; + let commands = children(2); + assert_eq!( + finish_task(&app, &initial, commands.clone()).await.0, + StatusCode::OK + ); + let first = child_poll(&app).await; + let second = child_poll(&app).await; + let closed = if close == "fail" { + finish_task(&app, &first, json!([{"type":"fail_workflow","message":"child failed λ","exception_type":"Boom","exception_class":"Boom"}])).await + } else { + request( + &app, + "POST", + &format!( + "/api/workflows/{}/{close}", + first["workflow_id"].as_str().unwrap() + ), + json!({"reason":"child closed λ"}), + ) + .await + }; + assert_eq!(closed.0, StatusCode::OK, "{}", closed.1); + assert_eq!(parent_tasks(&database, &run, "ready").await, 1); + let resumed = poll(&app, "parent-worker", "workflow").await; + assert_eq!(resumed["run_id"], run["run_id"]); + let event = resumed["history_events"] + .as_array() + .unwrap() + .last() + .unwrap(); + assert_eq!( + event["payload"]["child_call_id"], + scheduled(&resumed["history_events"])[0]["child_call_id"] + ); + assert!(event["payload"]["failure_id"].is_string()); + // A waiting acknowledgment lets the successful sibling test the closed + // barrier without a ready task accidentally coalescing a second wakeup. + let waiting = request(&app, "POST", &format!("/api/worker/workflow-tasks/{}/fail", resumed["task_id"].as_str().unwrap()), + json!({"lease_owner":resumed["lease_owner"],"workflow_task_attempt":resumed["workflow_task_attempt"], + "failure":{"type":"WorkflowTaskWaitingForHistory","message":"retaining failed barrier"}})).await; + assert_eq!(waiting.0, StatusCode::OK, "{}", waiting.1); + assert_eq!(finish_task(&app, &second, done(42)).await.0, StatusCode::OK); + assert!(poll(&app, "parent-worker", "workflow").await.is_null()); + let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + check_paths(&history, &commands); + assert_eq!(observation(&app, &run).await["summary_state"], "observed"); + runtime.close().await; + database.remove().await; + } +} + +#[tokio::test] +async fn incomplete_nested_mixed_selected_and_started_groups_leave_the_original_lease_unchanged() { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + workers(&app).await; + let run = start(&app, "refused-child-all").await; + let initial = poll(&app, "parent-worker", "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let mut cases = vec![json!([children(2)[0]])]; + for shape in [ + "duplicate", + "nested", + "mixed", + "select", + "started", + "scope", + "sequence", + ] { + let mut commands = children(2); + match shape { + "duplicate" => commands[1] = commands[0].clone(), + "nested" => { + let group = commands[0]["parallel_group_path"][0].clone(); + commands[0]["parallel_group_path"] + .as_array_mut() + .unwrap() + .push(group); + } + "mixed" => { + for command in commands.as_array_mut().unwrap() { + command["parallel_group_kind"] = json!("mixed"); + command["parallel_group_id"] = json!("parallel-calls:1:2"); + let mut entry = command.clone(); + entry.as_object_mut().unwrap().retain(|key, _| { + key.starts_with("parallel_group_") && key != "parallel_group_path" + }); + command["parallel_group_path"] = json!([entry]); + } + } + "select" => commands[0]["parallel_group_mode"] = json!("select"), + "started" => commands[0]["await"] = json!("started"), + "scope" => commands[0]["cancellation_scope_id"] = json!("unqualified-child-scope"), + "sequence" => { + for command in commands.as_array_mut().unwrap() { + command["parallel_group_base_sequence"] = json!(2); + command["parallel_group_id"] = json!("parallel-children:2:2"); + let mut entry = command.clone(); + entry.as_object_mut().unwrap().retain(|key, _| { + key.starts_with("parallel_group_") && key != "parallel_group_path" + }); + command["parallel_group_path"] = json!([entry]); + } + } + _ => unreachable!(), + } + cases.push(commands); + } + for commands in cases { + let denied = finish_task(&app, &initial, commands.clone()).await; + assert!( + matches!( + denied.0, + StatusCode::UNPROCESSABLE_ENTITY | StatusCode::CONFLICT + ), + "{commands}: {}", + denied.1 + ); + assert_eq!( + run_history(&app, &run["workflow_id"], &run["run_id"]).await, + before + ); + assert_eq!( + count( + &database, + "SELECT COUNT(*) FROM workflow_child_calls".into() + ) + .await, + 0 + ); + assert_eq!( + count(&database, "SELECT COUNT(*) FROM workflow_links".into()).await, + 0 + ); + assert_eq!(parent_tasks(&database, &run, "leased").await, 1); + assert!(child_poll(&app).await.is_null()); + } + assert_eq!( + finish_task(&app, &initial, children(2)).await.0, + StatusCode::OK + ); + runtime.close().await; + database.remove().await; +} + +#[tokio::test] +async fn retried_and_continued_members_keep_the_original_call_path_and_barrier() { + for transition in ["retry", "continue"] { + let database = TestDatabase::new().await; + let first = database.open().await.unwrap(); + let app = router(first.clone()); + workers(&app).await; + let run = start(&app, &format!("child-all-{transition}")).await; + let initial = poll(&app, "parent-worker", "workflow").await; + let mut commands = children(2); + if transition == "retry" { + commands[0]["retry_policy"] = json!({"max_attempts":2,"backoff_seconds":[0]}); + } + assert_eq!( + finish_task(&app, &initial, commands.clone()).await.0, + StatusCode::OK + ); + let prefix = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + let a = child_poll(&app).await; + let b = child_poll(&app).await; + let (member, sibling) = if a["run_id"] == scheduled(&prefix)[0]["child_workflow_run_id"] { + (a, b) + } else { + (b, a) + }; + assert_eq!( + finish_task(&app, &sibling, done(42)).await.0, + StatusCode::OK + ); + let next = if transition == "retry" { + json!([{"type":"fail_workflow","message":"retry group member"}]) + } else { + json!([{"type":"continue_as_new"}]) + }; + assert_eq!(finish_task(&app, &member, next).await.0, StatusCode::OK); + assert!(poll(&app, "parent-worker", "workflow").await.is_null()); + check_paths( + &run_history(&app, &run["workflow_id"], &run["run_id"]).await, + &commands, + ); + observed(&app, &run, 1).await; + drop(app); + first.close().await; + let cold = database.open().await.unwrap(); + let app = router(cold.clone()); + let replacement = child_poll(&app).await; + assert_eq!(replacement["workflow_id"], member["workflow_id"]); + assert_ne!(replacement["run_id"], member["run_id"]); + assert_eq!( + finish_task(&app, &replacement, done(9007199254740993)) + .await + .0, + StatusCode::OK + ); + let resumed = poll(&app, "parent-worker", "workflow").await; + assert_eq!(resumed["run_id"], run["run_id"]); + check_paths(&resumed["history_events"], &commands); + let last = resumed["history_events"] + .as_array() + .unwrap() + .last() + .unwrap(); + assert_eq!( + last["payload"]["child_call_id"], + scheduled(&prefix)[0]["child_call_id"] + ); + assert_eq!( + last["payload"]["child_workflow_run_id"], + replacement["run_id"] + ); + observed(&app, &run, 0).await; + assert_eq!( + finish_task(&app, &resumed, done(42)).await.0, + StatusCode::OK + ); + cold.close().await; + database.remove().await; + } +} + +#[tokio::test] +async fn one_and_full_batch_child_barriers_have_bounded_wait_inventory() { + for size in [1, 100] { + let database = TestDatabase::new().await; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + workers(&app).await; + let run = start(&app, &format!("bounded-children-{size}")).await; + let initial = poll(&app, "parent-worker", "workflow").await; + let admitted = finish_task(&app, &initial, children(size)).await; + assert_eq!(admitted.0, StatusCode::OK, "{}", admitted.1); + assert_eq!( + admitted.1["created_task_ids"].as_array().unwrap().len(), + size + ); + let summary = observation(&app, &run).await; + assert_eq!(summary["summary_state"], "observed", "{summary}"); + assert_eq!(summary["history_fan_out"], size); + assert_eq!(summary["current_waits"]["returned_count"], size.min(50)); + assert_eq!(summary["current_waits"]["has_more"], size > 50); + assert_eq!(parent_tasks(&database, &run, "ready").await, 0); + runtime.close().await; + database.remove().await; + } +} + +#[tokio::test] +async fn child_group_projection_failure_rolls_back_all_starts_and_original_lease() { + let database = TestDatabase::new().await; + let TestDatabase::Sqlite(_) = &database else { + return; + }; + let runtime = database.open().await.unwrap(); + let app = router(runtime.clone()); + workers(&app).await; + let run = start(&app, "atomic-child-group").await; + let initial = poll(&app, "parent-worker", "workflow").await; + let before = run_history(&app, &run["workflow_id"], &run["run_id"]).await; + database.execute("CREATE TRIGGER reject_second_child_wait BEFORE INSERT ON workflow_run_waits WHEN NEW.kind='child' AND NEW.sequence=2 BEGIN SELECT RAISE(ABORT,'child group fault'); END").await; + assert_eq!( + finish_task(&app, &initial, children(2)).await.0, + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + run_history(&app, &run["workflow_id"], &run["run_id"]).await, + before + ); + assert_eq!( + count( + &database, + "SELECT COUNT(*) FROM workflow_child_calls".into() + ) + .await, + 0 + ); + assert_eq!( + count(&database, "SELECT COUNT(*) FROM workflow_links".into()).await, + 0 + ); + assert_eq!( + count(&database, "SELECT COUNT(*) FROM workflow_instances".into()).await, + 1 + ); + assert_eq!(parent_tasks(&database, &run, "leased").await, 1); + assert!(child_poll(&app).await.is_null()); + database + .execute("DROP TRIGGER reject_second_child_wait") + .await; + assert_eq!( + finish_task(&app, &initial, children(2)).await.0, + StatusCode::OK + ); + observed(&app, &run, 2).await; + runtime.close().await; + database.remove().await; +} From 140fef18919464b28fd1e65815b1254ce2650e54 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:21:24 -0600 Subject: [PATCH 2/5] fix: restore child barrier patch placement --- rust/src/runtime/child_summary.rs | 2 +- rust/src/runtime/children.rs | 42 +++++++++++++++---------------- rust/src/runtime/continuations.rs | 10 ++++---- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/rust/src/runtime/child_summary.rs b/rust/src/runtime/child_summary.rs index e347ae75a..543bd64a6 100644 --- a/rust/src/runtime/child_summary.rs +++ b/rust/src/runtime/child_summary.rs @@ -134,10 +134,10 @@ impl ExpectedChild { } Some(Self { id, + parallel, run: text("child_workflow_run_id")?, instance: text("child_workflow_instance_id")?, label, - parallel, started: false, status: "waiting", latest_run: text("child_workflow_run_id")?, diff --git a/rust/src/runtime/children.rs b/rust/src/runtime/children.rs index 7db68ba6c..7649148ef 100644 --- a/rust/src/runtime/children.rs +++ b/rust/src/runtime/children.rs @@ -351,6 +351,8 @@ where ) -> Result { let parent_run = DB::string(parent, "id")?; let parent_instance = DB::string(parent, "workflow_instance_id")?; + let parallel = super::selections::metadata(command, "child")?; + super::selections::check_sequence(¶llel, sequence)?; // Published PHP starts a child whatever else the parent has open: // activities, timers, waits and other children. A replayed completion // is handled by the enclosing immutable task receipt. @@ -361,8 +363,6 @@ where text(command, "queue").map(str::to_owned) })?; let arguments = envelope(command, "arguments")?; - let parallel = super::selections::metadata(command, "child")?; - super::selections::check_sequence(¶llel, sequence)?; let policies = policies(command)?; let child_instance = id(); let child_run = id(); @@ -522,6 +522,10 @@ where )); } let link = &links[0]; + let parallel = super::selections::child_metadata(DB::optional_document_row( + link, + "parallel_group_path", + )?)?; let parent_run = DB::string(link, "parent_workflow_run_id")?; let sequence = DB::number(link, "sequence")?; let link_id = DB::string(link, "id")?; @@ -538,10 +542,6 @@ where if !matches!( DB::string(&parent, "status")?.as_str(), "pending" | "running" | "waiting" - let parallel = super::selections::child_metadata(DB::optional_document_row( - link, - "parallel_group_path", - )?)?; ) || DB::string(&parent, "current_run_id")? != parent_run { return Ok(()); @@ -649,7 +649,17 @@ where } } } + super::selections::merge(&mut payload, ¶llel); Self::append(tx, &parent_run, event_type, payload, None, None).await?; + // Record the outcome even while the all barrier remains closed. The + // last successful member observes every committed call under the same + // parent transition lock; any failed member wakes the parent at once. + if matches!(resolution, ChildResolution::Completed(_)) + && !Self::all_groups_closed(tx, &parent_run, ¶llel).await? + { + return Ok(()); + } + let parent_resume = Self::child_parent_resume(tx, &parent).await?; let (task_id, task_status) = match parent_resume { ChildParentResume::UpdateOnly => return Ok(()), ChildParentResume::Existing { id, status } => (id, status), @@ -668,17 +678,7 @@ where /// the same call. Returns whether a retry started. async fn start_child_retry( tx: &mut Transaction<'_, DB>, - super::selections::merge(&mut payload, ¶llel); parent: &DB::Row, - // Record the outcome even while the all barrier remains closed. The - // last successful member observes every committed call under the same - // parent transition lock; any failed member wakes the parent at once. - if matches!(resolution, ChildResolution::Completed(_)) - && !Self::all_groups_closed(tx, &parent_run, ¶llel).await? - { - return Ok(()); - } - let parent_resume = Self::child_parent_resume(tx, &parent).await?; link: &DB::Row, call: &DB::Row, failed: &DB::Row, @@ -687,6 +687,10 @@ where ) -> Result { let parent_run = DB::string(parent, "id")?; let failed_run = DB::string(failed, "id")?; + let parallel = super::selections::child_metadata(DB::optional_document_row( + link, + "parallel_group_path", + )?)?; // The policy the parent recorded when it scheduled the call, or the // call's own when that event is in an earlier run. let mut scheduled = Value::Null; @@ -706,10 +710,6 @@ where .filter(|policy| policy.is_object() || policy.is_array()) .unwrap_or(Value::Null) }; - let parallel = super::selections::child_metadata(DB::optional_document_row( - link, - "parallel_group_path", - )?)?; if policy.is_null() { return Ok(false); } @@ -794,6 +794,7 @@ where if let Some(fields) = started.as_object_mut() { fields.retain(|_, value| !value.is_null()); } + super::selections::merge(&mut started, ¶llel); Self::append(tx, &parent_run, "ChildRunStarted", started, None, None).await?; let mut child_started = json!({"workflow_class":workflow_class,"workflow_type":workflow_type,"workflow_instance_id":instance, "workflow_run_id":retry_run,"parent_workflow_instance_id":DB::string(parent, "workflow_instance_id")?, @@ -820,7 +821,6 @@ where metadata["last_retry_backoff_seconds"] = json!(backoff); Self::query("UPDATE workflow_child_calls SET resolved_child_instance_id=$1,resolved_child_run_id=$2,status='started',started_at=$3,metadata=$4,updated_at=$5 WHERE id=$6") .bind(&instance).bind(&retry_run).bind(DB::bind_time(at)).bind(DB::document(&metadata)).bind(DB::bind_time(at)) - super::selections::merge(&mut started, ¶llel); .bind(DB::number(call, "id")?).execute(&mut **tx).await?; tx.summary_changed(&parent_run); Ok(true) diff --git a/rust/src/runtime/continuations.rs b/rust/src/runtime/continuations.rs index af864e175..a87f0c647 100644 --- a/rust/src/runtime/continuations.rs +++ b/rust/src/runtime/continuations.rs @@ -330,6 +330,11 @@ where if call_id.is_none() { payload.as_object_mut().expect("an object").remove("child_call_id"); } + let parallel = super::selections::child_metadata(DB::optional_document_row( + &parent_link, + "parallel_group_path", + )?)?; + super::selections::merge(&mut payload, ¶llel); // PHP projects the parent again (`projectRun`). tx.summary_changed(&parent_run); Self::append(tx, &parent_run, "ChildRunStarted", payload, None, None).await?; @@ -356,11 +361,6 @@ where .map(|names| names.iter().filter_map(Value::as_str).collect()) .unwrap_or_default(); let mut messages = Vec::new(); - let parallel = super::selections::child_metadata(DB::optional_document_row( - &parent_link, - "parallel_group_path", - )?)?; - super::selections::merge(&mut payload, ¶llel); for row in Self::query("SELECT * FROM workflow_signal_records WHERE workflow_run_id=$1 AND target_scope='instance' AND status='received' AND closed_at IS NULL") .bind(old).fetch_all(&mut **tx).await? { From 2165de2dbcace35f6e605f9da575fe1719c6b989 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:33:02 -0600 Subject: [PATCH 3/5] fix: preserve terminated child barrier summaries --- rust/src/runtime/child_summary.rs | 4 ++-- rust/tests/execution/parallel_children.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/rust/src/runtime/child_summary.rs b/rust/src/runtime/child_summary.rs index 543bd64a6..f6f30248a 100644 --- a/rust/src/runtime/child_summary.rs +++ b/rust/src/runtime/child_summary.rs @@ -158,7 +158,7 @@ impl ExpectedChild { fn wait_status(&self) -> &'static str { match self.status { "waiting" => "open", - "cancelled" => "cancelled", + "cancelled" | "terminated" => "cancelled", _ => "resolved", } } @@ -241,7 +241,7 @@ where }; let pending = rows.iter().find(|row| { DB::string(row, "source_status").is_ok_and(|status| { - matches!(status.as_str(), "cancelled" | "failed") + matches!(status.as_str(), "cancelled" | "terminated" | "failed") || (status == "completed" && open.is_none()) }) }); diff --git a/rust/tests/execution/parallel_children.rs b/rust/tests/execution/parallel_children.rs index 5bcc9d9a9..193c8d362 100644 --- a/rust/tests/execution/parallel_children.rs +++ b/rust/tests/execution/parallel_children.rs @@ -299,7 +299,12 @@ async fn failed_cancelled_and_terminated_members_wake_immediately_without_reopen assert!(poll(&app, "parent-worker", "workflow").await.is_null()); let history = run_history(&app, &run["workflow_id"], &run["run_id"]).await; check_paths(&history, &commands); - assert_eq!(observation(&app, &run).await["summary_state"], "observed"); + let summary = observation(&app, &run).await; + assert_eq!(summary["summary_state"], "observed", "{close}: {summary}"); + assert_eq!( + summary["liveness_state"], "repair_needed", + "{close}: {summary}" + ); runtime.close().await; database.remove().await; } @@ -521,7 +526,7 @@ async fn child_group_projection_failure_rolls_back_all_starts_and_original_lease database.execute("CREATE TRIGGER reject_second_child_wait BEFORE INSERT ON workflow_run_waits WHEN NEW.kind='child' AND NEW.sequence=2 BEGIN SELECT RAISE(ABORT,'child group fault'); END").await; assert_eq!( finish_task(&app, &initial, children(2)).await.0, - StatusCode::INTERNAL_SERVER_ERROR + StatusCode::SERVICE_UNAVAILABLE ); assert_eq!( run_history(&app, &run["workflow_id"], &run["run_id"]).await, From 1c897420923be4caa1105f8d2b5f2ba97b1263b0 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:54:44 -0600 Subject: [PATCH 4/5] test: compare flat child all with published PHP SDK --- .github/workflows/server-parity.yml | 3 + .../conformance/server-parity/contract.mjs | 3 + .../parallel-child-all-contract.mjs | 142 + .../parallel-child-all-probe.php | 175 + scripts/conformance/server-parity/probe.php | 6 + .../ServerParity/parallel-child-all.json | 51 + .../parallel-child-all-php.json | 3162 +++++++++++++++++ ...rverParityParallelChildAllContractTest.mjs | 68 + 8 files changed, 3610 insertions(+) create mode 100644 scripts/conformance/server-parity/parallel-child-all-contract.mjs create mode 100644 scripts/conformance/server-parity/parallel-child-all-probe.php create mode 100644 tests/Fixtures/ServerParity/parallel-child-all.json create mode 100644 tests/Fixtures/ServerParityObservations/parallel-child-all-php.json create mode 100644 tests/Unit/ServerParityParallelChildAllContractTest.mjs diff --git a/.github/workflows/server-parity.yml b/.github/workflows/server-parity.yml index e51f2d4c0..cd6250238 100644 --- a/.github/workflows/server-parity.yml +++ b/.github/workflows/server-parity.yml @@ -28,6 +28,7 @@ on: - tests/Fixtures/HistoryBudget/** - tests/Fixtures/ServerUpgrade/** - tests/Fixtures/ServerParity/** + - tests/Fixtures/ServerParityObservations/** - tests/Fixtures/ServerParityBaselines/** - tests/Fixtures/ServerParityNormalization/** - tests/Unit/ServerParityRunnerTest.mjs @@ -61,6 +62,7 @@ on: - tests/Unit/ServerParityOperatorMetricsContractTest.mjs - tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs - tests/Unit/ServerParityConcurrentWaitsContractTest.mjs + - tests/Unit/ServerParityParallelChildAllContractTest.mjs - tests/Unit/ServerParityHistoryExportWaitsContractTest.mjs - tests/Unit/ServerParityContinueTransfersContractTest.mjs - tests/Unit/ServerParityLargeCommandBatchesContractTest.mjs @@ -2267,6 +2269,7 @@ jobs: tests/Unit/ServerParityOperatorMetricsContractTest.mjs \ tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs \ tests/Unit/ServerParityConcurrentWaitsContractTest.mjs \ + tests/Unit/ServerParityParallelChildAllContractTest.mjs \ tests/Unit/ServerParityHistoryExportWaitsContractTest.mjs \ tests/Unit/ServerParityContinueTransfersContractTest.mjs \ tests/Unit/ServerParityChildDescriptionsContractTest.mjs \ diff --git a/scripts/conformance/server-parity/contract.mjs b/scripts/conformance/server-parity/contract.mjs index 3d924d111..4300b1c6c 100644 --- a/scripts/conformance/server-parity/contract.mjs +++ b/scripts/conformance/server-parity/contract.mjs @@ -14,6 +14,7 @@ import {checkSearchAttributes} from './search-attribute-contract.mjs'; import {checkStorageTest} from './storage-test-contract.mjs'; import {checkActivityLeaseRenewal} from './activity-lease-renewal-contract.mjs'; import {checkConcurrentWaits} from './concurrent-waits-contract.mjs'; +import {checkParallelChildAll} from './parallel-child-all-contract.mjs'; import {checkHistoryExportWaits} from './history-export-waits-contract.mjs'; import {checkContinueTransfers} from './continue-transfers-contract.mjs'; import {checkLargeCommandBatches} from './large-command-batches-contract.mjs'; @@ -194,6 +195,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { const storageTest = fixture.storage_test ? checkStorageTest(fixture, observation, workflowId) : null; const activityLeaseRenewal = fixture.activity_lease_renewal ? checkActivityLeaseRenewal(fixture, observation, workflowId) : null; const concurrentWaits = fixture.concurrent_waits ? checkConcurrentWaits(fixture, observation) : null; + const parallelChildAll = fixture.parallel_child_all ? checkParallelChildAll(fixture, observation) : null; const historyExportWaits = fixture.history_export_waits ? checkHistoryExportWaits(fixture, observation) : null; const continueTransfers = fixture.continue_transfers ? checkContinueTransfers(fixture, observation) : null; const largeCommandBatches = fixture.large_command_batches ? checkLargeCommandBatches(fixture, observation) : null; @@ -792,6 +794,7 @@ export function checkObservation(fixture, observation, workflowId, artifacts) { ...(storageTest ? {storage_test: storageTest} : {}), ...(activityLeaseRenewal ? {activity_lease_renewal: activityLeaseRenewal} : {}), ...(concurrentWaits ? {concurrent_waits: concurrentWaits} : {}), + ...(parallelChildAll ? {parallel_child_all: parallelChildAll} : {}), ...(historyExportWaits ? {history_export_waits: historyExportWaits} : {}), ...(continueTransfers ? {continue_transfers: continueTransfers} : {}), ...(largeCommandBatches ? {large_command_batches: largeCommandBatches} : {}), diff --git a/scripts/conformance/server-parity/parallel-child-all-contract.mjs b/scripts/conformance/server-parity/parallel-child-all-contract.mjs new file mode 100644 index 000000000..d96e14090 --- /dev/null +++ b/scripts/conformance/server-parity/parallel-child-all-contract.mjs @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; + +const equal = assert.deepStrictEqual; +const groupFields = ['parallel_group_id', 'parallel_group_kind', 'parallel_group_base_sequence', + 'parallel_group_size', 'parallel_group_index']; +const map = value => ({type: 'map', value}); +const list = value => ({type: 'list', value}); +const text = value => ({type: 'string', value}); +const nonempty = value => assert.ok(typeof value === 'string' && value.length > 0, 'canonical identity'); + +// The immutable PHP reference refuses duplicate completed tasks. Native's +// existing immutable receipt acknowledges them. Both must preserve all history. +function duplicate(outcome, phase) { + equal(outcome, outcome.status === 409 + ? {status: 409, recorded: null, reason: phase === 'start' ? 'task_not_leased' : 'run_closed', message: null} + : {status: 200, recorded: false, reason: 'already_completed', message: null}, 'reviewed duplicate outcome'); +} + +function history(events, expected) { + equal(events.map(event => event.event_type), expected, 'complete canonical event inventory'); + equal(events.map(event => event.sequence), events.map((_, index) => index + 1), 'canonical event positions'); + for (const event of events) { + assert.ok(!Object.hasOwn(event.payload, 'task'), 'task claims are excluded from observations'); + } +} + +function metadata(payload, size, index) { + const group = {parallel_group_id: `parallel-children:1:${size}`, parallel_group_kind: 'child', + parallel_group_base_sequence: 1, parallel_group_size: size, parallel_group_index: index}; + for (const field of groupFields) equal(payload[field], group[field], 'original child barrier descriptor'); + equal(payload.parallel_group_path, [group], 'original flat child barrier path'); + equal(payload.parallel_group_mode ?? 'all', 'all', 'all barrier mode'); +} + +export function checkParallelChildAll(fixture, observation) { + const definition = fixture.parallel_child_all; + const state = observation.parallel_child_all; + const projection = {scope: 'parallel_child_all', cases: Object.keys(definition.cases).length}; + if (observation.mode === 'embedded') { + equal(state, {applicable: false, reason: 'embedded_has_no_worker_protocol'}, 'explicit embedded boundary'); + return projection; + } + equal(state?.applicable, true, 'real worker-protocol observation'); + equal(Object.keys(state.cases).sort(), Object.keys(definition.cases).sort(), 'complete case inventory'); + for (const [name, size] of Object.entries(definition.cases)) { + const value = state.cases[name]; + const success = name === 'single' || name === 'out_of_order'; + const terminal = {failed: 'ChildRunFailed', cancelled: 'ChildRunCancelled', terminated: 'ChildRunTerminated'}[name]; + const prefix = ['StartAccepted', 'WorkflowStarted', ...Array.from({length: size}, () => ['ChildWorkflowScheduled', 'ChildRunStarted']).flat()]; + const order = success && size === 3 ? [2, 0, 1] : Array.from({length: size}, (_, index) => index); + equal(value.size, size, 'authored group size'); + equal(value.initial_recorded, true, 'atomic initial commit'); + equal(value.initial_no_parent_task, true, 'no committed-start wakeup'); + equal(value.completion_order, order, 'actual controlled completion order'); + equal(value.empty_before_last, success ? Array(size - 1).fill(true) : [], 'no early successful wakeup'); + equal(value.no_extra_parent_task, true, 'no duplicate parent resume'); + duplicate(value.duplicate_start_outcome, 'start'); + duplicate(value.duplicate_finish_outcome, 'finish'); + equal(value.duplicate_child_outcomes.length, success ? size : size - 1, 'every successful child duplicate observed'); + value.duplicate_child_outcomes.forEach(outcome => duplicate(outcome, 'child')); + history(value.prefix, prefix); + history(value.events, [...prefix, ...(success ? Array(size).fill('ChildRunCompleted') : [terminal]), 'WorkflowCompleted']); + equal(value.events.slice(0, prefix.length), value.prefix, 'immutable original child start history'); + equal(value.events, value.at_parent_close, 'later abandoned siblings add no parent history'); + nonempty(value.workflow_id); + nonempty(value.run_id); + equal(value.prefix[1].payload.workflow_instance_id, value.workflow_id, 'original parent instance'); + equal(value.prefix[1].payload.workflow_run_id, value.run_id, 'original parent run'); + equal(value.initial_commands.length, size, 'official Replayer authors every member together'); + equal(value.children.length, size, 'complete original child inventory'); + const identities = [value.workflow_id, value.run_id]; + const outputs = []; + for (let index = 0; index < size; index++) { + const command = value.initial_commands[index]; + const child = value.children[index]; + const scheduled = value.prefix[2 + index * 2].payload; + const started = value.prefix[3 + index * 2].payload; + const argument = map({index: {type: 'int64', value: String(index)}, value: fixture.typed_value}); + const output = map({echo: argument}); + outputs.push(output); + equal(command.type, 'start_child_workflow', 'real SDK child command'); + equal(command.workflow_type, definition.child_type, 'registered child type'); + equal(command.queue, `${value.workflow_id}-children`, 'isolated child queue'); + equal(command.typed_arguments, list([argument]), 'exact SDK child argument types'); + assert.ok(!Object.hasOwn(command, 'await'), 'deferred result barrier uses the published default'); + metadata(command, size, index); + for (const payload of [scheduled, started]) { + metadata(payload, size, index); + equal(payload.sequence, index + 1, 'authored child sequence'); + equal(payload.child_workflow_class, definition.child_type, 'original child class'); + equal(payload.child_workflow_type, definition.child_type, 'original child type'); + equal(payload.parent_close_policy, 'abandon', 'default parent-close policy'); + equal(payload.cancellation_policy, 'abandon', 'default cancellation policy'); + } + for (const key of ['workflow_link_id', 'child_call_id', 'child_workflow_instance_id', 'child_workflow_run_id']) { + nonempty(scheduled[key]); + equal(started[key], scheduled[key], `child start keeps ${key}`); + } + equal(scheduled.workflow_link_id, scheduled.child_call_id, 'original call is original link'); + equal(child.workflow_id, scheduled.child_workflow_instance_id, 'actual child instance'); + equal(child.run_id, scheduled.child_workflow_run_id, 'actual child run'); + identities.push(scheduled.child_call_id, child.workflow_id, child.run_id); + equal(child.typed_arguments, list([argument]), 'actual claimed child argument types'); + const childSuccess = success || index !== 0; + const childEvents = childSuccess ? ['WorkflowStarted', 'WorkflowCompleted'] + : name === 'failed' ? ['WorkflowStarted', 'WorkflowFailed'] + : name === 'cancelled' ? ['WorkflowStarted', 'CancelRequested', 'WorkflowCancelled'] + : ['WorkflowStarted', 'TerminateRequested', 'WorkflowTerminated']; + history(child.events, childEvents); + for (const [key, expected] of Object.entries({workflow_instance_id: child.workflow_id, + workflow_run_id: child.run_id, parent_workflow_instance_id: value.workflow_id, + parent_workflow_run_id: value.run_id, parent_sequence: index + 1, + workflow_link_id: scheduled.child_call_id, child_call_id: scheduled.child_call_id})) { + equal(child.events[0].payload[key], expected, `canonical original child ${key}`); + } + if (childSuccess) equal(child.events.at(-1).typed_decoded.output, output, 'canonical child output types'); + } + equal(new Set(identities).size, identities.length, 'all original child/call/run identities are distinct'); + const resolutions = value.events.slice(prefix.length, -1); + for (const [offset, event] of resolutions.entries()) { + const index = success ? order[offset] : 0; + const scheduled = value.prefix[2 + index * 2].payload; + metadata(event.payload, size, index); + equal(event.payload.sequence, index + 1, 'resolution retains original authored sequence'); + for (const key of ['workflow_link_id', 'child_call_id', 'child_workflow_instance_id', 'child_workflow_run_id']) { + equal(event.payload[key], scheduled[key], `resolution retains ${key}`); + } + equal(event.payload.child_status, success ? 'completed' : name, 'canonical child outcome'); + if (success) { + equal(event.typed_decoded.result, outputs[index], 'canonical typed child resolution'); + equal(event.typed_decoded.output, outputs[index], 'canonical typed child output'); + } else nonempty(event.payload.failure_id); + } + const result = success ? map({results: list(outputs)}) : map({ + failure_class: text('DurableWorkflow\\Exception\\ChildWorkflowFailed'), failure_type: text(terminal), + workflow_type: text(definition.child_type), message: text(definition.failure_messages[name]), + }); + equal(value.typed_result, result, 'cold official Replayer preserves result order or typed failure'); + equal(value.events.at(-1).typed_decoded.output, result, 'canonical committed parent output'); + } + return projection; +} diff --git a/scripts/conformance/server-parity/parallel-child-all-probe.php b/scripts/conformance/server-parity/parallel-child-all-probe.php new file mode 100644 index 000000000..c39d437c6 --- /dev/null +++ b/scripts/conformance/server-parity/parallel-child-all-probe.php @@ -0,0 +1,175 @@ +createNamespace($namespace, 'flat child all barriers', 30); + $stack = HandlerStack::create(); + $stack->push(LockPressurePollRetries::middleware(), 'lock_pressure_poll_retry'); + $http = new ChildAllHttp(['base_uri' => rtrim($url, '/'), 'http_errors' => false, + 'connect_timeout' => 1, 'timeout' => 30, 'handler' => $stack]); + $raw = static function (string $method, string $path, ?array $body = null, bool $allowConflict = false) use ($http, $namespace): array { + $options = ['headers' => array_filter([ + 'Authorization' => getenv('DW_PARITY_TOKEN') ? 'Bearer '.getenv('DW_PARITY_TOKEN') : null, + 'Accept' => 'application/json', 'X-Namespace' => $namespace, + 'X-Durable-Workflow-Control-Plane-Version' => '2', 'X-Durable-Workflow-Protocol-Version' => '1.20', + ])]; + if ($body !== null) { + $options['json'] = $body === [] ? (object) [] : $body; + } + $response = $http->request($method, $path, $options); + $value = json_decode((string) $response->getBody(), true, flags: JSON_THROW_ON_ERROR); + if ($response->getStatusCode() >= 400 && ! ($allowConflict && $response->getStatusCode() === 409)) { + throw new RuntimeException('Child all '.$method.' '.$path.' HTTP '.$response->getStatusCode() + .': '.($value['reason'] ?? $value['message'] ?? 'unknown')); + } + + return $value + ['observed_http_status' => $response->getStatusCode()]; + }; + $codec = $admin->payloadCodec(); + $frame = static fn (mixed $value): array => ['codec' => 'avro', 'blob' => $codec->encode($value)]; + $history = static function (string $id, string $run) use ($raw, $codec): array { + $events = $raw('GET', '/api/workflows/'.rawurlencode($id).'/runs/'.$run.'/history')['events'] ?? []; + $events = decodeHistory($events, $codec->decodeEnvelope(...)); + + return array_map(static function (array $event): array { + $event = array_intersect_key($event, array_flip(['event_type', 'sequence', 'payload', 'typed_decoded'])); + // PHP canonical events include a task snapshot with lease authority. + // The child contract needs call/link/run identities, never that claim. + unset($event['payload']['task']); + + return $event; + }, $events); + }; + $complete = static fn (array $task, array $commands, bool $duplicate = false): array => $raw('POST', + '/api/worker/workflow-tasks/'.$task['task_id'].'/complete', [ + 'lease_owner' => $task['lease_owner'], 'workflow_task_attempt' => $task['workflow_task_attempt'], + 'commands' => $commands, + ], $duplicate); + $duplicateOutcome = static fn (array $response): array => ['status' => $response['observed_http_status'], + 'recorded' => $response['recorded'] ?? null, 'reason' => $response['reason'] ?? null, + 'message' => $response['message'] ?? null]; + $cases = []; + foreach ($definition['cases'] as $case => $size) { + $id = $workflowId.'-'.$case; + $parentQueue = $id.'-parents'; + $childQueue = $id.'-children'; + $parentWorker = $id.'-parent-worker'; + $childWorker = $id.'-child-worker'; + foreach ([[$parentWorker, $parentQueue, $definition['parent_type']], [$childWorker, $childQueue, $definition['child_type']]] as [$worker, $queue, $type]) { + $raw('POST', '/api/worker/register', ['worker_id' => $worker, 'task_queue' => $queue, + 'runtime' => 'php', 'supported_workflow_types' => [$type], 'supported_activity_types' => [], + 'capability_manifest' => array_fill_keys(['local_activities', 'worker_sessions', 'sticky_execution'], + ['supported' => false, 'minimum_protocol_version' => '1.18']), + ]); + } + $poll = static fn (bool $parent): ?array => $raw('POST', '/api/worker/workflow-tasks/poll', [ + 'worker_id' => $parent ? $parentWorker : $childWorker, + 'task_queue' => $parent ? $parentQueue : $childQueue, 'timeout_seconds' => 0, + ])['task'] ?? null; + $run = $raw('POST', '/api/workflows', ['workflow_id' => $id, 'workflow_type' => $definition['parent_type'], + 'task_queue' => $parentQueue, 'input' => $frame([$fixture['input']])]); + $initial = $poll(true) ?? throw new RuntimeException('Child all missing initial parent task.'); + $workflow = static function (WorkflowContext $context, array $value) use ($size, $definition, $childQueue): array { + try { + return ['results' => $context->all(array_map(static fn (int $index) => $context->deferChildWorkflow( + $definition['child_type'], [['index' => $index, 'value' => $value]], ['queue' => $childQueue], + ), range(0, $size - 1)))]; + } catch (ChildWorkflowFailed $error) { + return ['failure_class' => $error::class, 'failure_type' => $error->failureType, + 'workflow_type' => $error->workflowType, 'message' => $error->getMessage()]; + } + }; + $replay = static fn (array $task, callable $handler): array => (new Replayer($codec)) + ->replay($handler, $task['history_events'], $codec->decodeEnvelope($task['arguments']), $task['task_queue']) + ->commands; + $commands = $replay($initial, $workflow); + $admitted = $complete($initial, $commands); + $prefix = $history($id, $run['run_id']); + $duplicate = $complete($initial, $commands, true); + if ($history($id, $run['run_id']) !== $prefix) { + throw new RuntimeException('Child all duplicate parent completion changed history.'); + } + $scheduled = array_values(array_filter($prefix, static fn (array $event): bool => $event['event_type'] === 'ChildWorkflowScheduled')); + if (count($scheduled) !== $size) { + throw new RuntimeException('Child all did not atomically start every child.'); + } + $tasks = []; + for ($index = 0; $index < $size; $index++) { + $task = $poll(false) ?? throw new RuntimeException('Child all missing child task.'); + $position = array_search($task['run_id'], array_column(array_column($scheduled, 'payload'), 'child_workflow_run_id'), true); + if ($position === false || isset($tasks[$position])) { + throw new RuntimeException('Child all claimed an unknown or duplicate child.'); + } + $tasks[$position] = $task; + } + ksort($tasks); + $state = ['size' => $size, 'workflow_id' => $id, 'run_id' => $run['run_id'], + 'initial_recorded' => $admitted['recorded'], 'duplicate_start_outcome' => $duplicateOutcome($duplicate), + 'initial_no_parent_task' => $poll(true) === null, + 'initial_commands' => array_map(static function (array $command) use ($codec): array { + $value = $command; + $value['typed_arguments'] = typedValue($codec->decodeEnvelope($command['arguments'])); + unset($value['arguments']); + + return $value; + }, $commands), 'prefix' => $prefix, 'children' => [], 'empty_before_last' => []]; + $success = in_array($case, ['single', 'out_of_order'], true); + $order = $success && $size === 3 ? [2, 0, 1] : range(0, $size - 1); + $state['completion_order'] = $order; + foreach ($order as $offset => $index) { + $task = $tasks[$index]; + $childWorkflow = static fn (WorkflowContext $context, array $value): array => ['echo' => $value]; + $childCommands = $replay($task, $childWorkflow); + if (! $success && $offset === 0) { + if ($case === 'failed') { + $childCommands = [['type' => 'fail_workflow', 'message' => 'child barrier failure λ', + 'exception_type' => 'ChildAllProbeFailure', 'exception_class' => 'ChildAllProbeFailure']]; + $complete($task, $childCommands); + } else { + $raw('POST', '/api/workflows/'.$task['workflow_id'].'/'.($case === 'cancelled' ? 'cancel' : 'terminate'), + ['reason' => 'child barrier closed λ']); + } + } else { + $complete($task, $childCommands); + $state['duplicate_child_outcomes'][] = $duplicateOutcome($complete($task, $childCommands, true)); + } + $state['children'][$index] = ['workflow_id' => $task['workflow_id'], 'run_id' => $task['run_id'], + 'typed_arguments' => typedValue($codec->decodeEnvelope($task['arguments'])), + 'events' => $history($task['workflow_id'], $task['run_id'])]; + if ($success && $offset < $size - 1) { + $state['empty_before_last'][] = $poll(true) === null; + } + if (($success && $offset === $size - 1) || (! $success && $offset === 0)) { + $resumed = $poll(true) ?? throw new RuntimeException('Child all missing eligible parent resume.'); + $parentCommands = $replay($resumed, $workflow); + if (count($parentCommands) !== 1 || $parentCommands[0]['type'] !== 'complete_workflow') { + throw new RuntimeException('Child all replay did not complete with committed results or typed failure.'); + } + $state['typed_result'] = typedValue($codec->decodeEnvelope($parentCommands[0]['result'])); + $complete($resumed, $parentCommands); + $state['duplicate_finish_outcome'] = $duplicateOutcome($complete($resumed, $parentCommands, true)); + $state['at_parent_close'] = $history($id, $run['run_id']); + } + } + ksort($state['children']); + $state['children'] = array_values($state['children']); + $state['events'] = $history($id, $run['run_id']); + $state['no_extra_parent_task'] = $poll(true) === null; + $cases[$case] = $state; + } + + return ['applicable' => true, 'cases' => $cases]; +} diff --git a/scripts/conformance/server-parity/probe.php b/scripts/conformance/server-parity/probe.php index efcc62855..216a89655 100644 --- a/scripts/conformance/server-parity/probe.php +++ b/scripts/conformance/server-parity/probe.php @@ -432,6 +432,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, $storageTest = isset($fixture['storage_test']) ? finishHttpStorageTest($client, $fixture, $workflowId, $url) : null; $activityLeaseRenewal = isset($fixture['activity_lease_renewal']) ? finishHttpActivityLeaseRenewal($client, $fixture, $workflowId, $url) : null; $concurrentWaits = isset($fixture['concurrent_waits']) ? finishHttpConcurrentWaits($client, $fixture, $workflowId, $url) : null; + $parallelChildAll = isset($fixture['parallel_child_all']) ? finishHttpParallelChildAll($client, $fixture, $workflowId, $url) : null; $historyExportWaits = isset($fixture['history_export_waits']) ? finishHttpHistoryExportWaits($client, $fixture, $workflowId, $url) : null; $continueTransfers = isset($fixture['continue_transfers']) ? finishHttpContinueTransfers($client, $fixture, $workflowId, $url) : null; $largeCommandBatches = isset($fixture['large_command_batches']) ? finishHttpLargeCommandBatches($client, $fixture, $workflowId, $url) : null; @@ -583,6 +584,7 @@ function httpObservation(array $fixture, string $workflowId, string $namespace, 'storage_test' => $storageTest, 'activity_lease_renewal' => $activityLeaseRenewal, 'concurrent_waits' => $concurrentWaits, + 'parallel_child_all' => $parallelChildAll, 'history_export_waits' => $historyExportWaits, 'continue_transfers' => $continueTransfers, 'large_command_batches' => $largeCommandBatches, @@ -940,6 +942,7 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa require __DIR__.'/storage-test-probe.php'; require __DIR__.'/activity-lease-renewal-probe.php'; require __DIR__.'/concurrent-waits-probe.php'; + require __DIR__.'/parallel-child-all-probe.php'; require __DIR__.'/history-export-waits-probe.php'; require __DIR__.'/continue-transfers-probe.php'; require __DIR__.'/large-command-batches-probe.php'; @@ -1147,6 +1150,9 @@ function embeddedObservation(array $fixture, string $workflowId, string $namespa if ($mode === 'embedded' && isset($fixture['storage_test'])) { $observation['storage_test'] = ['applicable' => false, 'reason' => 'embedded_has_no_http_storage_diagnostic']; } + if ($mode === 'embedded' && isset($fixture['parallel_child_all'])) { + $observation['parallel_child_all'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; + } if ($mode === 'embedded' && isset($fixture['concurrent_waits'])) { $observation['concurrent_waits'] = ['applicable' => false, 'reason' => 'embedded_has_no_worker_protocol']; } diff --git a/tests/Fixtures/ServerParity/parallel-child-all.json b/tests/Fixtures/ServerParity/parallel-child-all.json new file mode 100644 index 000000000..038ae5e64 --- /dev/null +++ b/tests/Fixtures/ServerParity/parallel-child-all.json @@ -0,0 +1,51 @@ +{ + "schema": "durable-workflow.server-parity-fixture/v1", + "review_state": "reviewed", + "workflow_type": "parity.v1.echo", + "input": { + "message": "concurrent waits λ", + "count": 9007199254740993, + "optional": null + }, + "typed_value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + }, + "activity": false, + "expected_events": [ + "StartAccepted", + "WorkflowStarted", + "WorkflowCompleted" + ], + "id": "parallel-child-all", + "contract": "Worker-protocol flat child-only all barriers authored and cold-replayed by the real published PHP SDK: every child starts atomically, carries the original call/link/instance/run and path, and returns exact ordered typed results despite out-of-order completion. Success waits for every member; failure/cancel/terminate wakes the parent immediately as ChildWorkflowFailed. Duplicate parent/child completions add no starts or history. After a failed parent catches its outcome and closes under default abandon, later successful siblings add no parent resolution or wakeup. This worker-HTTP side contract is explicitly inapplicable to embedded execution. Native cold database replacement, retries/continuation, malformed and unqualified shape refusals, bounded inventory and rollback are covered separately by cross-backend native tests. Child select/nested/mixed/scoped/started-in-all remain refused.", + "parallel_child_all": { + "parent_type": "parity.v1.child_all_parent", + "child_type": "parity.v1.child_all_child", + "cases": { + "single": 1, + "out_of_order": 3, + "failed": 2, + "cancelled": 2, + "terminated": 2 + }, + "failure_messages": { + "failed": "child barrier failure λ", + "cancelled": "Workflow cancelled: child barrier closed λ", + "terminated": "Workflow terminated: child barrier closed λ" + } + } +} diff --git a/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json b/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json new file mode 100644 index 000000000..4b1542c56 --- /dev/null +++ b/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json @@ -0,0 +1,3162 @@ +{ + "evidence": { + "reference_image": "ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07", + "sdk_php": "3.9.1", + "sdk_php_source": "e4d77aade4ddc02eb0ba40f827709b8b4817725e", + "wire_probe_sha256": "cc6acc99a1fe5a718f2f8658449561336e31738f5cc34c97422808e0f0c33131", + "type_annotation": "Offline decode of the retained canonical Avro frames with the identical installed SDK codec; no additional runtime I/O.", + "redaction": "Canonical task snapshots omitted; nondeterministic IDs replaced consistently; only checked payload fields retained.", + "deduplication": "Unit baseline reconstructs duplicate prefix/at-parent-close arrays from the already checked immutable canonical events; runtime observations retain each independently." + }, + "parallel_child_all": { + "applicable": true, + "cases": { + "single": { + "size": 1, + "workflow_id": "{parent:single}", + "run_id": "{parent-run:single}", + "initial_recorded": true, + "duplicate_start_outcome": { + "status": 409, + "recorded": null, + "reason": "task_not_leased", + "message": null + }, + "initial_no_parent_task": true, + "initial_commands": [ + { + "type": "start_child_workflow", + "queue": "{parent:single}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + } + ], + "children": [ + { + "workflow_id": "{child:single:0}", + "run_id": "{child-run:single:0}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:single:0}", + "workflow_run_id": "{child-run:single:0}", + "parent_workflow_instance_id": "{parent:single}", + "parent_workflow_run_id": "{parent-run:single}", + "parent_sequence": 1, + "workflow_link_id": "{call:single:0}", + "child_call_id": "{call:single:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + } + ], + "empty_before_last": [], + "completion_order": [ + 0 + ], + "duplicate_child_outcomes": [ + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + } + ], + "typed_result": { + "type": "map", + "value": { + "results": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + ] + } + } + }, + "duplicate_finish_outcome": { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + "events": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_instance_id": "{parent:single}", + "workflow_run_id": "{parent-run:single}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{parent:single}", + "workflow_run_id": "{parent-run:single}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:single:0}", + "child_call_id": "{call:single:0}", + "child_workflow_instance_id": "{child:single:0}", + "child_workflow_run_id": "{child-run:single:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 4, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:single:0}", + "child_call_id": "{call:single:0}", + "child_workflow_instance_id": "{child:single:0}", + "child_workflow_run_id": "{child-run:single:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 5, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:single:0}", + "child_call_id": "{call:single:0}", + "child_workflow_instance_id": "{child:single:0}", + "child_workflow_run_id": "{child-run:single:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "completed", + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:1", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 1, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + "result": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + }, + { + "sequence": 6, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "results": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + ] + } + } + } + } + } + ], + "no_extra_parent_task": true + }, + "out_of_order": { + "size": 3, + "workflow_id": "{parent:out_of_order}", + "run_id": "{parent-run:out_of_order}", + "initial_recorded": true, + "duplicate_start_outcome": { + "status": 409, + "recorded": null, + "reason": "task_not_leased", + "message": null + }, + "initial_no_parent_task": true, + "initial_commands": [ + { + "type": "start_child_workflow", + "queue": "{parent:out_of_order}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + }, + { + "type": "start_child_workflow", + "queue": "{parent:out_of_order}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + }, + { + "type": "start_child_workflow", + "queue": "{parent:out_of_order}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + } + ], + "children": [ + { + "workflow_id": "{child:out_of_order:0}", + "run_id": "{child-run:out_of_order:0}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:out_of_order:0}", + "workflow_run_id": "{child-run:out_of_order:0}", + "parent_workflow_instance_id": "{parent:out_of_order}", + "parent_workflow_run_id": "{parent-run:out_of_order}", + "parent_sequence": 1, + "workflow_link_id": "{call:out_of_order:0}", + "child_call_id": "{call:out_of_order:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + }, + { + "workflow_id": "{child:out_of_order:1}", + "run_id": "{child-run:out_of_order:1}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:out_of_order:1}", + "workflow_run_id": "{child-run:out_of_order:1}", + "parent_workflow_instance_id": "{parent:out_of_order}", + "parent_workflow_run_id": "{parent-run:out_of_order}", + "parent_sequence": 2, + "workflow_link_id": "{call:out_of_order:1}", + "child_call_id": "{call:out_of_order:1}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + }, + { + "workflow_id": "{child:out_of_order:2}", + "run_id": "{child-run:out_of_order:2}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:out_of_order:2}", + "workflow_run_id": "{child-run:out_of_order:2}", + "parent_workflow_instance_id": "{parent:out_of_order}", + "parent_workflow_run_id": "{parent-run:out_of_order}", + "parent_sequence": 3, + "workflow_link_id": "{call:out_of_order:2}", + "child_call_id": "{call:out_of_order:2}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + } + ], + "empty_before_last": [ + true, + true + ], + "completion_order": [ + 2, + 0, + 1 + ], + "duplicate_child_outcomes": [ + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + } + ], + "typed_result": { + "type": "map", + "value": { + "results": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + ] + } + } + }, + "duplicate_finish_outcome": { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + "events": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_instance_id": "{parent:out_of_order}", + "workflow_run_id": "{parent-run:out_of_order}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{parent:out_of_order}", + "workflow_run_id": "{parent-run:out_of_order}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:out_of_order:0}", + "child_call_id": "{call:out_of_order:0}", + "child_workflow_instance_id": "{child:out_of_order:0}", + "child_workflow_run_id": "{child-run:out_of_order:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 4, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:out_of_order:0}", + "child_call_id": "{call:out_of_order:0}", + "child_workflow_instance_id": "{child:out_of_order:0}", + "child_workflow_run_id": "{child-run:out_of_order:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 5, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:out_of_order:1}", + "child_call_id": "{call:out_of_order:1}", + "child_workflow_instance_id": "{child:out_of_order:1}", + "child_workflow_run_id": "{child-run:out_of_order:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 6, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:out_of_order:1}", + "child_call_id": "{call:out_of_order:1}", + "child_workflow_instance_id": "{child:out_of_order:1}", + "child_workflow_run_id": "{child-run:out_of_order:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 7, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 3, + "workflow_link_id": "{call:out_of_order:2}", + "child_call_id": "{call:out_of_order:2}", + "child_workflow_instance_id": "{child:out_of_order:2}", + "child_workflow_run_id": "{child-run:out_of_order:2}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 8, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 3, + "workflow_link_id": "{call:out_of_order:2}", + "child_call_id": "{call:out_of_order:2}", + "child_workflow_instance_id": "{child:out_of_order:2}", + "child_workflow_run_id": "{child-run:out_of_order:2}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 9, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 3, + "workflow_link_id": "{call:out_of_order:2}", + "child_call_id": "{call:out_of_order:2}", + "child_workflow_instance_id": "{child:out_of_order:2}", + "child_workflow_run_id": "{child-run:out_of_order:2}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "completed", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 2 + } + ] + }, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + "result": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + }, + { + "sequence": 10, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:out_of_order:0}", + "child_call_id": "{call:out_of_order:0}", + "child_workflow_instance_id": "{child:out_of_order:0}", + "child_workflow_run_id": "{child-run:out_of_order:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "completed", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + "result": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + }, + { + "sequence": 11, + "event_type": "ChildRunCompleted", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:out_of_order:1}", + "child_call_id": "{call:out_of_order:1}", + "child_workflow_instance_id": "{child:out_of_order:1}", + "child_workflow_run_id": "{child-run:out_of_order:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "completed", + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:3", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 3, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + "result": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + }, + { + "sequence": 12, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "results": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + }, + { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "2" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + ] + } + } + } + } + } + ], + "no_extra_parent_task": true + }, + "failed": { + "size": 2, + "workflow_id": "{parent:failed}", + "run_id": "{parent-run:failed}", + "initial_recorded": true, + "duplicate_start_outcome": { + "status": 409, + "recorded": null, + "reason": "task_not_leased", + "message": null + }, + "initial_no_parent_task": true, + "initial_commands": [ + { + "type": "start_child_workflow", + "queue": "{parent:failed}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + }, + { + "type": "start_child_workflow", + "queue": "{parent:failed}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + } + ], + "children": [ + { + "workflow_id": "{child:failed:0}", + "run_id": "{child-run:failed:0}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:failed:0}", + "workflow_run_id": "{child-run:failed:0}", + "parent_workflow_instance_id": "{parent:failed}", + "parent_workflow_run_id": "{parent-run:failed}", + "parent_sequence": 1, + "workflow_link_id": "{call:failed:0}", + "child_call_id": "{call:failed:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowFailed", + "payload": { + "failure_id": "{failure:failed}" + }, + "typed_decoded": [] + } + ] + }, + { + "workflow_id": "{child:failed:1}", + "run_id": "{child-run:failed:1}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:failed:1}", + "workflow_run_id": "{child-run:failed:1}", + "parent_workflow_instance_id": "{parent:failed}", + "parent_workflow_run_id": "{parent-run:failed}", + "parent_sequence": 2, + "workflow_link_id": "{call:failed:1}", + "child_call_id": "{call:failed:1}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + } + ], + "empty_before_last": [], + "completion_order": [ + 0, + 1 + ], + "typed_result": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunFailed" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "child barrier failure λ" + } + } + }, + "duplicate_finish_outcome": { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + "duplicate_child_outcomes": [ + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + } + ], + "events": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_instance_id": "{parent:failed}", + "workflow_run_id": "{parent-run:failed}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{parent:failed}", + "workflow_run_id": "{parent-run:failed}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:failed:0}", + "child_call_id": "{call:failed:0}", + "child_workflow_instance_id": "{child:failed:0}", + "child_workflow_run_id": "{child-run:failed:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 4, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:failed:0}", + "child_call_id": "{call:failed:0}", + "child_workflow_instance_id": "{child:failed:0}", + "child_workflow_run_id": "{child-run:failed:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 5, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:failed:1}", + "child_call_id": "{call:failed:1}", + "child_workflow_instance_id": "{child:failed:1}", + "child_workflow_run_id": "{child-run:failed:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 6, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:failed:1}", + "child_call_id": "{call:failed:1}", + "child_workflow_instance_id": "{child:failed:1}", + "child_workflow_run_id": "{child-run:failed:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 7, + "event_type": "ChildRunFailed", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:failed:0}", + "child_call_id": "{call:failed:0}", + "child_workflow_instance_id": "{child:failed:0}", + "child_workflow_run_id": "{child-run:failed:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "failed", + "failure_id": "{failure:failed}", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 8, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunFailed" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "child barrier failure λ" + } + } + } + } + } + ], + "no_extra_parent_task": true + }, + "cancelled": { + "size": 2, + "workflow_id": "{parent:cancelled}", + "run_id": "{parent-run:cancelled}", + "initial_recorded": true, + "duplicate_start_outcome": { + "status": 409, + "recorded": null, + "reason": "task_not_leased", + "message": null + }, + "initial_no_parent_task": true, + "initial_commands": [ + { + "type": "start_child_workflow", + "queue": "{parent:cancelled}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + }, + { + "type": "start_child_workflow", + "queue": "{parent:cancelled}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + } + ], + "children": [ + { + "workflow_id": "{child:cancelled:0}", + "run_id": "{child-run:cancelled:0}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:cancelled:0}", + "workflow_run_id": "{child-run:cancelled:0}", + "parent_workflow_instance_id": "{parent:cancelled}", + "parent_workflow_run_id": "{parent-run:cancelled}", + "parent_sequence": 1, + "workflow_link_id": "{call:cancelled:0}", + "child_call_id": "{call:cancelled:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "CancelRequested", + "payload": { + "workflow_instance_id": "{child:cancelled:0}", + "workflow_run_id": "{child-run:cancelled:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "WorkflowCancelled", + "payload": { + "workflow_instance_id": "{child:cancelled:0}", + "workflow_run_id": "{child-run:cancelled:0}", + "failure_id": "{failure:cancelled}" + }, + "typed_decoded": [] + } + ] + }, + { + "workflow_id": "{child:cancelled:1}", + "run_id": "{child-run:cancelled:1}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:cancelled:1}", + "workflow_run_id": "{child-run:cancelled:1}", + "parent_workflow_instance_id": "{parent:cancelled}", + "parent_workflow_run_id": "{parent-run:cancelled}", + "parent_sequence": 2, + "workflow_link_id": "{call:cancelled:1}", + "child_call_id": "{call:cancelled:1}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + } + ], + "empty_before_last": [], + "completion_order": [ + 0, + 1 + ], + "typed_result": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunCancelled" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "Workflow cancelled: child barrier closed λ" + } + } + }, + "duplicate_finish_outcome": { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + "duplicate_child_outcomes": [ + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + } + ], + "events": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_instance_id": "{parent:cancelled}", + "workflow_run_id": "{parent-run:cancelled}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{parent:cancelled}", + "workflow_run_id": "{parent-run:cancelled}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:cancelled:0}", + "child_call_id": "{call:cancelled:0}", + "child_workflow_instance_id": "{child:cancelled:0}", + "child_workflow_run_id": "{child-run:cancelled:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 4, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:cancelled:0}", + "child_call_id": "{call:cancelled:0}", + "child_workflow_instance_id": "{child:cancelled:0}", + "child_workflow_run_id": "{child-run:cancelled:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 5, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:cancelled:1}", + "child_call_id": "{call:cancelled:1}", + "child_workflow_instance_id": "{child:cancelled:1}", + "child_workflow_run_id": "{child-run:cancelled:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 6, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:cancelled:1}", + "child_call_id": "{call:cancelled:1}", + "child_workflow_instance_id": "{child:cancelled:1}", + "child_workflow_run_id": "{child-run:cancelled:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 7, + "event_type": "ChildRunCancelled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:cancelled:0}", + "child_call_id": "{call:cancelled:0}", + "child_workflow_instance_id": "{child:cancelled:0}", + "child_workflow_run_id": "{child-run:cancelled:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "cancelled", + "failure_id": "{failure:cancelled}", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 8, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunCancelled" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "Workflow cancelled: child barrier closed λ" + } + } + } + } + } + ], + "no_extra_parent_task": true + }, + "terminated": { + "size": 2, + "workflow_id": "{parent:terminated}", + "run_id": "{parent-run:terminated}", + "initial_recorded": true, + "duplicate_start_outcome": { + "status": 409, + "recorded": null, + "reason": "task_not_leased", + "message": null + }, + "initial_no_parent_task": true, + "initial_commands": [ + { + "type": "start_child_workflow", + "queue": "{parent:terminated}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + }, + { + "type": "start_child_workflow", + "queue": "{parent:terminated}-children", + "workflow_type": "parity.v1.child_all_child", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ], + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + } + } + ], + "children": [ + { + "workflow_id": "{child:terminated:0}", + "run_id": "{child-run:terminated:0}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "0" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:terminated:0}", + "workflow_run_id": "{child-run:terminated:0}", + "parent_workflow_instance_id": "{parent:terminated}", + "parent_workflow_run_id": "{parent-run:terminated}", + "parent_sequence": 1, + "workflow_link_id": "{call:terminated:0}", + "child_call_id": "{call:terminated:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "TerminateRequested", + "payload": { + "workflow_instance_id": "{child:terminated:0}", + "workflow_run_id": "{child-run:terminated:0}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "WorkflowTerminated", + "payload": { + "workflow_instance_id": "{child:terminated:0}", + "workflow_run_id": "{child-run:terminated:0}", + "failure_id": "{failure:terminated}" + }, + "typed_decoded": [] + } + ] + }, + { + "workflow_id": "{child:terminated:1}", + "run_id": "{child-run:terminated:1}", + "typed_arguments": { + "type": "list", + "value": [ + { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + ] + }, + "events": [ + { + "sequence": 1, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{child:terminated:1}", + "workflow_run_id": "{child-run:terminated:1}", + "parent_workflow_instance_id": "{parent:terminated}", + "parent_workflow_run_id": "{parent-run:terminated}", + "parent_sequence": 2, + "workflow_link_id": "{call:terminated:1}", + "child_call_id": "{call:terminated:1}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "echo": { + "type": "map", + "value": { + "index": { + "type": "int64", + "value": "1" + }, + "value": { + "type": "map", + "value": { + "message": { + "type": "string", + "value": "concurrent waits λ" + }, + "count": { + "type": "int64", + "value": "9007199254740993" + }, + "optional": { + "type": "null", + "value": null + } + } + } + } + } + } + } + } + } + ] + } + ], + "empty_before_last": [], + "completion_order": [ + 0, + 1 + ], + "typed_result": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunTerminated" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "Workflow terminated: child barrier closed λ" + } + } + }, + "duplicate_finish_outcome": { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + }, + "duplicate_child_outcomes": [ + { + "status": 409, + "recorded": null, + "reason": "run_closed", + "message": null + } + ], + "events": [ + { + "sequence": 1, + "event_type": "StartAccepted", + "payload": { + "workflow_instance_id": "{parent:terminated}", + "workflow_run_id": "{parent-run:terminated}" + }, + "typed_decoded": [] + }, + { + "sequence": 2, + "event_type": "WorkflowStarted", + "payload": { + "workflow_instance_id": "{parent:terminated}", + "workflow_run_id": "{parent-run:terminated}" + }, + "typed_decoded": [] + }, + { + "sequence": 3, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:terminated:0}", + "child_call_id": "{call:terminated:0}", + "child_workflow_instance_id": "{child:terminated:0}", + "child_workflow_run_id": "{child-run:terminated:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 4, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:terminated:0}", + "child_call_id": "{call:terminated:0}", + "child_workflow_instance_id": "{child:terminated:0}", + "child_workflow_run_id": "{child-run:terminated:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 5, + "event_type": "ChildWorkflowScheduled", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:terminated:1}", + "child_call_id": "{call:terminated:1}", + "child_workflow_instance_id": "{child:terminated:1}", + "child_workflow_run_id": "{child-run:terminated:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 6, + "event_type": "ChildRunStarted", + "payload": { + "sequence": 2, + "workflow_link_id": "{call:terminated:1}", + "child_call_id": "{call:terminated:1}", + "child_workflow_instance_id": "{child:terminated:1}", + "child_workflow_run_id": "{child-run:terminated:1}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "parent_close_policy": "abandon", + "cancellation_policy": "abandon", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 1 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 7, + "event_type": "ChildRunTerminated", + "payload": { + "sequence": 1, + "workflow_link_id": "{call:terminated:0}", + "child_call_id": "{call:terminated:0}", + "child_workflow_instance_id": "{child:terminated:0}", + "child_workflow_run_id": "{child-run:terminated:0}", + "child_workflow_class": "parity.v1.child_all_child", + "child_workflow_type": "parity.v1.child_all_child", + "child_run_number": 1, + "child_status": "terminated", + "failure_id": "{failure:terminated}", + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0, + "parallel_group_path": [ + { + "parallel_group_id": "parallel-children:1:2", + "parallel_group_kind": "child", + "parallel_group_base_sequence": 1, + "parallel_group_size": 2, + "parallel_group_index": 0 + } + ] + }, + "typed_decoded": [] + }, + { + "sequence": 8, + "event_type": "WorkflowCompleted", + "payload": {}, + "typed_decoded": { + "output": { + "type": "map", + "value": { + "failure_class": { + "type": "string", + "value": "DurableWorkflow\\Exception\\ChildWorkflowFailed" + }, + "failure_type": { + "type": "string", + "value": "ChildRunTerminated" + }, + "workflow_type": { + "type": "string", + "value": "parity.v1.child_all_child" + }, + "message": { + "type": "string", + "value": "Workflow terminated: child barrier closed λ" + } + } + } + } + } + ], + "no_extra_parent_task": true + } + } + } +} diff --git a/tests/Unit/ServerParityParallelChildAllContractTest.mjs b/tests/Unit/ServerParityParallelChildAllContractTest.mjs new file mode 100644 index 000000000..9d9b4bd19 --- /dev/null +++ b/tests/Unit/ServerParityParallelChildAllContractTest.mjs @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import test from 'node:test'; +import {checkParallelChildAll} from '../../scripts/conformance/server-parity/parallel-child-all-contract.mjs'; + +const fixture = JSON.parse(readFileSync(new URL('../Fixtures/ServerParity/parallel-child-all.json', import.meta.url), 'utf8')); +const reference = JSON.parse(readFileSync(new URL('../Fixtures/ServerParityObservations/parallel-child-all-php.json', import.meta.url), 'utf8')); +function observation() { + const state = structuredClone(reference.parallel_child_all); + for (const value of Object.values(state.cases)) { + value.prefix = structuredClone(value.events.slice(0, 2 + value.size * 2)); + value.at_parent_close = structuredClone(value.events); + } + return {mode: 'http', parallel_child_all: state}; +} + +test('the redacted published PHP observation satisfies the five-case contract', () => { + assert.deepStrictEqual(checkParallelChildAll(fixture, observation()), {scope: 'parallel_child_all', cases: 5}); +}); + +test('the explicitly reviewed native duplicate representation has the same common projection', () => { + const value = observation(); + for (const item of Object.values(value.parallel_child_all.cases)) { + const duplicate = {status: 200, recorded: false, reason: 'already_completed', message: null}; + item.duplicate_start_outcome = structuredClone(duplicate); + item.duplicate_finish_outcome = structuredClone(duplicate); + item.duplicate_child_outcomes = item.duplicate_child_outcomes.map(() => structuredClone(duplicate)); + } + assert.deepStrictEqual(checkParallelChildAll(fixture, value), {scope: 'parallel_child_all', cases: 5}); +}); + +test('embedded execution has an explicit worker-protocol boundary', () => { + assert.deepStrictEqual(checkParallelChildAll(fixture, {mode: 'embedded', + parallel_child_all: {applicable: false, reason: 'embedded_has_no_worker_protocol'}}), + {scope: 'parallel_child_all', cases: 5}); + assert.throws(() => checkParallelChildAll(fixture, {mode: 'embedded', parallel_child_all: {applicable: true}})); +}); + +for (const [name, mutate] of [ + ['a missing case', state => { delete state.cases.single; }], + ['an extra case', state => { state.cases.extra = {}; }], + ['an initial start turn', state => { state.cases.single.initial_no_parent_task = false; }], + ['an early successful wakeup', state => { state.cases.out_of_order.empty_before_last[0] = false; }], + ['a missing atomic group member', state => { state.cases.out_of_order.initial_commands.pop(); }], + ['a mixed child group', state => { state.cases.single.initial_commands[0].parallel_group_kind = 'mixed'; }], + ['a changed durable path', state => { state.cases.single.prefix[2].payload.parallel_group_path = []; }], + ['a child started handle', state => { state.cases.single.initial_commands[0].await = 'started'; }], + ['a wrong child queue', state => { state.cases.single.initial_commands[0].queue = 'other'; }], + ['a changed int64 argument', state => { state.cases.single.children[0].typed_arguments.value[0].value.value.value.count.value = '9007199254740992'; }], + ['a replacement original child run', state => { state.cases.single.children[0].run_id = 'other-run'; }], + ['a wrong original parent', state => { state.cases.single.children[0].events[0].payload.parent_workflow_run_id = 'other-parent'; }], + ['a wrong child call resolution', state => { state.cases.single.events[4].payload.child_call_id = 'other-call'; }], + ['a swapped ordered result', state => { state.cases.out_of_order.typed_result.value.results.value.reverse(); }], + ['a wrong canonical committed output', state => { state.cases.single.events.at(-1).typed_decoded.output = {type: 'null', value: null}; }], + ['a missing failure identity', state => { state.cases.failed.events.at(-2).payload.failure_id = ''; }], + ['an untyped failed-child result', state => { state.cases.failed.typed_result.value.failure_type.value = 'Error'; }], + ['a changed failure diagnostic', state => { state.cases.cancelled.typed_result.value.message.value = 'different'; }], + ['a missing child closure', state => { state.cases.terminated.children[0].events.pop(); }], + ['a later abandoned child changing parent history', state => { state.cases.failed.at_parent_close.pop(); }], + ['a duplicate start recording effects', state => { state.cases.single.duplicate_start_outcome = {status: 200, recorded: true, reason: null, message: null}; }], + ['an unexplained duplicate refusal', state => { state.cases.single.duplicate_child_outcomes[0].reason = 'other'; }], + ['an extra parent resume', state => { state.cases.single.no_extra_parent_task = false; }], + ['a recorded task claim', state => { state.cases.single.prefix[2].payload.task = {lease_owner: 'fixture-authority'}; }], +]) test(`the comparison rejects ${name}`, () => { + const value = observation(); + mutate(value.parallel_child_all); + assert.throws(() => checkParallelChildAll(fixture, value)); +}); From 427b01de809190ea6355e0a0e29bc7ab97fbe6f6 Mon Sep 17 00:00:00 2001 From: Richard McDaniel <1130888+rmcdaniel@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:03:15 -0600 Subject: [PATCH 5/5] test: bind child all baseline to committed reference probe --- .../ServerParityObservations/parallel-child-all-php.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json b/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json index 4b1542c56..54482b135 100644 --- a/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json +++ b/tests/Fixtures/ServerParityObservations/parallel-child-all-php.json @@ -3,10 +3,13 @@ "reference_image": "ghcr.io/durable-workflow/server@sha256:980fe0a052089ced34ec3fcd2be2ea4e77436861d8fb537fb6654cb652a53e07", "sdk_php": "3.9.1", "sdk_php_source": "e4d77aade4ddc02eb0ba40f827709b8b4817725e", - "wire_probe_sha256": "cc6acc99a1fe5a718f2f8658449561336e31738f5cc34c97422808e0f0c33131", - "type_annotation": "Offline decode of the retained canonical Avro frames with the identical installed SDK codec; no additional runtime I/O.", + "wire_probe_sha256": "5ed97b9b31ae0dd7aec0e1193e0d6b24bcd0ad463419f2e25ce58c0f19cf4bbc", + "type_annotation": "Canonical Avro type annotations captured with the installed SDK codec by the committed HTTP observer.", "redaction": "Canonical task snapshots omitted; nondeterministic IDs replaced consistently; only checked payload fields retained.", - "deduplication": "Unit baseline reconstructs duplicate prefix/at-parent-close arrays from the already checked immutable canonical events; runtime observations retain each independently." + "deduplication": "Unit baseline reconstructs duplicate prefix/at-parent-close arrays from the already checked immutable canonical events; runtime observations retain each independently.", + "observer_revision": "1c897420923be4caa1105f8d2b5f2ba97b1263b0", + "observed_started_at": "2026-11-04T12:00:49.434562+00:00", + "observed_finished_at": "2026-11-04T12:01:28.603546+00:00" }, "parallel_child_all": { "applicable": true,