diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index e5df5d26fe..be7b4a95d7 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1650,7 +1650,7 @@ export interface components { run_id?: string | null; status: string; }; - /** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */ + /** @description A current snapshot node's declared metadata and latest build, which the node listing leaves to node-states. */ CatalogNodeRecord: { /** @description The checks on the node by their newest result. */ checks: components["schemas"]["CatalogNodeChecks"]; @@ -2275,7 +2275,7 @@ export interface components { kind: "sensor_evaluation"; outcome: components["schemas"]["CustomSensorOutcome"]; } | { - /** @description Model diff text the document left out to stay under its cap; stored apart, never read back here. */ + /** @description Model diff text, which the document never carries; stored apart, never read back here. */ diffs?: { [key: string]: components["schemas"]["PlanModelDiff"]; }; @@ -2739,7 +2739,7 @@ export interface components { /** @enum {string} */ status: "unavailable"; }; - /** @description A planned model's diff text, from the plan document or, where the document left it out, apart from it. */ + /** @description A planned model's diff text, stored beside its plan document. */ PlanDiffRead: components["schemas"]["PlanModelDiff"] & { model: string; }; @@ -5643,7 +5643,7 @@ export interface operations { parameters: { query?: never; header?: { - /** @description An ETag from an earlier response; 304 when the snapshot and state version are unchanged */ + /** @description An ETag from an earlier response; 304 when the snapshot is unchanged */ "If-None-Match"?: string | null; }; path: { @@ -5654,10 +5654,10 @@ export interface operations { }; requestBody?: never; responses: { - /** @description Every node of the current snapshot; empty before the first publication */ + /** @description Every node of the current snapshot without its state, which node-states serves; empty before the first publication */ 200: { headers: { - /** @description ":"; absent before the first publication */ + /** @description ""; absent before the first publication */ ETag?: string; [name: string]: unknown; }; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs index 997216342e..ece3e9b482 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use serde_json::Value; +use serde_json::{Value, json}; use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs; use sqlbuild_orchestration::platform::constants::{PLAN_DIFF_BYTES, PLAN_DOCUMENT_BYTES}; use sqlbuild_orchestration::platform::models::PlanModelDiff; @@ -11,7 +11,7 @@ use crate::native_commands::_helpers::location::execution::{plan_failure, withou use crate::native_commands::constants::{PLAN_NOT_PRODUCED, PLAN_OUTPUT_BYTES}; use crate::native_commands::types::LocationPlan; -/// The plan `sqb plan --json` printed; over the cap its model diffs leave it, keeping their counts, and travel beside it. +/// The plan `sqb plan --json` printed; its model diffs leave it, keeping their counts, and travel beside it. pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) -> LocationPlan { if output.len() > PLAN_OUTPUT_BYTES { return without_diffs(plan_failure("the plan is too large to show")); @@ -20,28 +20,16 @@ pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) -> Ok(document) if document.is_object() && planned => document, _ => return without_diffs(plan_failure(PLAN_NOT_PRODUCED)), }; - if output.len() <= PLAN_DOCUMENT_BYTES { - return without_diffs(document); - } let (document, detached) = detach_model_diffs(document); if !serde_json::to_vec(&document).is_ok_and(|bytes| bytes.len() <= PLAN_DOCUMENT_BYTES) { return without_diffs(plan_failure("the plan is too large to show")); } - let diffs: BTreeMap = detached - .into_iter() - .map(|(model, diff)| { - ( - model, - PlanModelDiff { - query_diff: diff.query_diff, - config_diff: diff.config_diff, - }, - ) - }) - .collect(); - if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) { - (document, diffs) - } else { - without_diffs(document) + match serde_json::from_value::>(json!(detached)) { + Ok(diffs) + if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) => + { + (document, diffs) + } + _ => without_diffs(document), } } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs index 480949e3af..7059286714 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs @@ -9,17 +9,20 @@ use crate::native_commands::tests::helpers::plan_output; use crate::native_commands::tests::test_types::{PlanDocumentTestCase, WorkCheckTestCase}; #[test] -fn given_plan_output_when_reading_it_on_a_location_then_only_oversized_diffs_leave_the_document() { +fn given_plan_output_when_reading_it_on_a_location_then_diffs_always_travel_beside_the_document() { let test_cases = [ PlanDocumentTestCase { - description: "a plan under the cap keeps its diffs", + description: "a small plan's diffs leave the document with their counts too", output: plan_output("+SELECT 2", 2), planned: true, - expected_first_model: json!({"name": "model_0", "reason": "query_changed", "query_diff": ["+SELECT 2"], "config_diff": ["+a"]}), - expected_diffs: (0, None), + expected_first_model: json!({"name": "model_0", "reason": "query_changed", "diff_stats": { + "query": {"added": 1, "removed": 0, "whitespace_only": false}, + "config": {"added": 1, "removed": 0, "whitespace_only": false}, + }}), + expected_diffs: (2, Some(vec!["+SELECT 2".to_owned()])), }, PlanDocumentTestCase { - description: "diffs over the cap leave the document with their counts and travel beside it", + description: "a plan over the document cap keeps every diff beside it", output: plan_output( &format!("+{}", "x".repeat(1024)), PLAN_DOCUMENT_BYTES / 1024 + 1, diff --git a/crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs b/crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs index 8d3163ef57..480ab3d780 100644 --- a/crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs +++ b/crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs @@ -2,28 +2,31 @@ use std::collections::BTreeMap; -use serde_json::Value; +use serde_json::{Map, Value}; use crate::plan_display::_helpers::diff_lines::config_change; use crate::plan_display::_helpers::plan_pages::{lines, row_diff_stats, text}; use crate::plan_display::constants::{CONFIG_CHANGE_FIELD, DIFF_STATS_FIELD, MODEL_DIFF_FIELDS}; -use crate::plan_display::models::PlanModelDiff; -/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and the text by model. -pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap) { - let mut diffs: BTreeMap = BTreeMap::new(); +/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and by model the `query_diff` and `config_diff` fields taken out. +pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap) { + let mut diffs: BTreeMap = BTreeMap::new(); let models = document.get_mut("models").and_then(Value::as_array_mut); for model in models.into_iter().flatten() { - let diff = PlanModelDiff { - query_diff: lines(model, "query_diff"), - config_diff: lines(model, "config_diff"), - }; - if diff.query_diff.is_none() && diff.config_diff.is_none() { + let mut diff: Map = Map::new(); + for field in MODEL_DIFF_FIELDS { + if let Some(text) = lines(model, field) { + diff.insert(field.to_owned(), Value::from(text)); + } + } + if diff.is_empty() { continue; } let stats = row_diff_stats(model); - let change = diff.config_diff.as_deref().and_then(config_change); - diffs.insert(text(model, "name").to_owned(), diff); + let change = lines(model, "config_diff") + .as_deref() + .and_then(config_change); + diffs.insert(text(model, "name").to_owned(), Value::Object(diff)); if let Some(fields) = model.as_object_mut() { for field in MODEL_DIFF_FIELDS { fields.remove(field); diff --git a/crates/sqlbuild-core/src/plan_display/main/mod.rs b/crates/sqlbuild-core/src/plan_display/main/mod.rs index f6c9622bf6..7e862d98a7 100644 --- a/crates/sqlbuild-core/src/plan_display/main/mod.rs +++ b/crates/sqlbuild-core/src/plan_display/main/mod.rs @@ -11,6 +11,5 @@ pub mod model_execution_annotation; pub mod model_materialization_label; pub mod model_resource_type; pub mod old_name_rows; -pub mod plan_model_diff; pub mod plan_section_page; pub mod plan_summary; diff --git a/crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs b/crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs deleted file mode 100644 index 72af85f666..0000000000 --- a/crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! One planned model's diff text, read from its plan document. - -use serde_json::Value; - -use crate::plan_display::_helpers::plan_pages::{lines, text}; -use crate::plan_display::models::PlanModelDiff; - -/// The named model's query and config diff text; `None` when the document has neither for it. -pub fn plan_model_diff(document: &Value, model: &str) -> Option { - let row = document - .get("models")? - .as_array()? - .iter() - .find(|row| text(row, "name") == model)?; - let diff = PlanModelDiff { - query_diff: lines(row, "query_diff"), - config_diff: lines(row, "config_diff"), - }; - (diff.query_diff.is_some() || diff.config_diff.is_some()).then_some(diff) -} diff --git a/crates/sqlbuild-core/src/plan_display/models.rs b/crates/sqlbuild-core/src/plan_display/models.rs index 6707c45711..51244eb2fb 100644 --- a/crates/sqlbuild-core/src/plan_display/models.rs +++ b/crates/sqlbuild-core/src/plan_display/models.rs @@ -94,10 +94,3 @@ pub struct DiffStats { /// The removed and added lines read the same with every whitespace character taken out. pub whitespace_only: bool, } - -/// A planned model's diff text, as `sqb plan` prints it. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct PlanModelDiff { - pub query_diff: Option>, - pub config_diff: Option>, -} diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs index 225e604ef0..bf67a9aed7 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs @@ -2,7 +2,6 @@ use serde_json::{Value, json}; use crate::plan_display::_helpers::diff_lines::diff_stats; use crate::plan_display::main::detach_model_diffs::detach_model_diffs; -use crate::plan_display::main::plan_model_diff::plan_model_diff; use crate::plan_display::main::plan_section_page::plan_section_page; use crate::plan_display::main::plan_summary::plan_summary; use crate::plan_display::models::{DiffStats, PlanPageSection}; @@ -591,20 +590,28 @@ fn given_a_document_whose_diffs_were_detached_when_paged_then_counts_remain_and_ let page = |document: &Value| { plan_section_page(document, PlanPageSection::QueryChanged, ((0, 50), None)).rows }; + let model_diff = |document: &Value| { + document["models"] + .as_array() + .into_iter() + .flatten() + .find(|row| row["name"] == test_case.model) + .and_then(|row| row.get("query_diff").cloned()) + }; assert_eq!( ( page(&detached), diffs .get(test_case.model) - .and_then(|diff| diff.query_diff.clone()), - plan_model_diff(&detached, test_case.model), - plan_model_diff(&original, test_case.model).and_then(|diff| diff.query_diff), + .and_then(|diff| diff.get("query_diff").cloned()), + model_diff(&detached), + model_diff(&original), ), ( page(&original), - test_case.expected_detached.map(owned), + test_case.expected_detached.map(|lines| json!(lines)), None, - test_case.expected_detached.map(owned), + test_case.expected_detached.map(|lines| json!(lines)), ), "{}: pages read the same counts, and the text leaves the document whole", test_case.description diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs index 4ba6bdebf7..8d507a5077 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs @@ -110,7 +110,7 @@ async fn stream_changes( (location_id, target): (Uuid, String), ) -> Result { let receiver = feed.sender.subscribe(); - let updates = watch_node_states(&cache, (location_id, &target)).await?; + let (updates, watch) = watch_node_states(&cache, (location_id, &target)).await?; let state = ChangeStream { store, subscriber, @@ -119,6 +119,7 @@ async fn stream_changes( target, receiver, updates, + _watch: watch, stopped: feed.stopped, keepalive: tokio::time::interval(Duration::from_secs(15)), initial: true, diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs index 032cc5426a..d1fcf9304c 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs @@ -84,9 +84,9 @@ pub(crate) async fn run_nodes( #[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes", operation_id = "listPersonCatalogNodes", params(("location_id" = Uuid, Path), ("target" = String, Path), - ("If-None-Match" = Option, Header, description = "An ETag from an earlier response; 304 when the snapshot and state version are unchanged")), - responses((status = 200, body = Vec, description = "Every node of the current snapshot; empty before the first publication", - headers(("ETag" = String, description = "\":\"; absent before the first publication"))), + ("If-None-Match" = Option, Header, description = "An ETag from an earlier response; 304 when the snapshot is unchanged")), + responses((status = 200, body = Vec, description = "Every node of the current snapshot without its state, which node-states serves; empty before the first publication", + headers(("ETag" = String, description = "\"\"; absent before the first publication"))), (status = 304, description = "Unchanged since the If-None-Match ETag", headers(("ETag" = String))), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] diff --git a/crates/sqlbuild-orchestration/src/api/models.rs b/crates/sqlbuild-orchestration/src/api/models.rs index 31c54c5e53..871c0e3400 100644 --- a/crates/sqlbuild-orchestration/src/api/models.rs +++ b/crates/sqlbuild-orchestration/src/api/models.rs @@ -575,6 +575,8 @@ pub(crate) struct ChangeStream { pub(crate) receiver: tokio::sync::broadcast::Receiver, pub(crate) updates: tokio::sync::broadcast::Receiver, + /// Keeps the target's node states refreshed while the stream is open. + pub(crate) _watch: crate::node_states::models::NodeStatesWatch, pub(crate) stopped: tokio::sync::watch::Receiver, pub(crate) keepalive: tokio::time::Interval, pub(crate) initial: bool, diff --git a/crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs b/crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs index 4d4ed3209c..f955db2b7d 100644 --- a/crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs +++ b/crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs @@ -1,4 +1,4 @@ -//! Response bodies are serialized once per view, off the async workers, and tagged with the view's snapshot and version. +//! Response bodies are serialized once, off the async workers: the node list once per snapshot, which also tags it, and the states once per view. use std::sync::Arc; @@ -6,63 +6,47 @@ use axum::body::Bytes; use crate::node_states::models::{CachedJson, NodeStatesRecord, TargetView}; use crate::store::errors::ControlError; -use crate::warehouse_metadata::models::CatalogNodeRecord; -/// The view's node list with each node's state; an empty list before the first publication. +/// The snapshot's node list without states; an empty list before the first publication. pub(crate) async fn nodes_json(view: &Arc) -> Result { + let catalog = view.catalog.clone(); let body = view - .nodes_body - .get_or_try_init(|| serialized(view.clone(), node_records)) + .catalog + .body + .get_or_try_init(|| serialized(move || serde_json::to_vec(&catalog.records))) .await? .clone(); Ok(CachedJson { etag: view .catalog .snapshot_id - .map(|snapshot_id| format!("\"{snapshot_id}:{}\"", view.version)), + .map(|snapshot_id| format!("\"{snapshot_id}\"")), body, }) } /// The view's states with its snapshot and version. pub(crate) async fn states_json(view: &Arc) -> Result { + let held = view.clone(); let body = view .states_body - .get_or_try_init(|| serialized(view.clone(), state_record)) + .get_or_try_init(|| serialized(move || serde_json::to_vec(&state_record(&held)))) .await? .clone(); Ok(CachedJson { etag: None, body }) } -async fn serialized( - view: Arc, - record: fn(&TargetView) -> T, +async fn serialized( + write: impl FnOnce() -> serde_json::Result> + Send + 'static, ) -> Result { tokio::task::spawn_blocking(move || { - serde_json::to_vec(&record(&view)) + write() .map(Bytes::from) .map_err(|error| ControlError::Invalid(error.to_string())) }) .await? } -fn node_records(view: &TargetView) -> Vec { - view.catalog - .records - .iter() - .map(|record| { - let mut record = record.clone(); - if let Some(state) = view.states.get(&record.node_key) { - record.last_build.clone_from(&state.last_build); - record.last_outcome.clone_from(&state.last_outcome); - record.definition_changed = state.definition_changed; - record.checks.clone_from(&state.checks); - } - record - }) - .collect() -} - fn state_record(view: &TargetView) -> NodeStatesRecord { NodeStatesRecord { snapshot_id: view.catalog.snapshot_id, diff --git a/crates/sqlbuild-orchestration/src/node_states/_helpers/reads.rs b/crates/sqlbuild-orchestration/src/node_states/_helpers/reads.rs index 0c56b0fa48..c319ace18d 100644 --- a/crates/sqlbuild-orchestration/src/node_states/_helpers/reads.rs +++ b/crates/sqlbuild-orchestration/src/node_states/_helpers/reads.rs @@ -117,6 +117,7 @@ fn snapshot_catalog(snapshot_id: Uuid, nodes: BTreeMap) -> snapshot_id: Some(snapshot_id), records, inputs, + body: Default::default(), } } diff --git a/crates/sqlbuild-orchestration/src/node_states/_helpers/views.rs b/crates/sqlbuild-orchestration/src/node_states/_helpers/views.rs index 040e61695f..20038aec53 100644 --- a/crates/sqlbuild-orchestration/src/node_states/_helpers/views.rs +++ b/crates/sqlbuild-orchestration/src/node_states/_helpers/views.rs @@ -11,25 +11,40 @@ use crate::api::models::ChangeNotice; use crate::node_states::_helpers::reads::read_target; use crate::node_states::constants::MAX_PUSHED_NODE_STATES; use crate::node_states::models::{ - NodeState, NodeStateCache, NodeStatusUpdate, TargetCache, TargetView, + NodeState, NodeStateCache, NodeStatesWatch, NodeStatusUpdate, TargetCache, TargetView, }; use crate::store::errors::ControlError; /// The target's cache entry, created empty on first use. -pub(crate) fn target_cache( +pub(crate) fn target_cache(cache: &NodeStateCache, scope: (Uuid, &str)) -> Arc { + held_entry(cache, scope, false) +} + +/// Watch the target's entry; counted under the targets lock, so a signal never drops an entry a stream holds. +pub(crate) fn watch_target(cache: &NodeStateCache, scope: (Uuid, &str)) -> NodeStatesWatch { + NodeStatesWatch { + entry: held_entry(cache, scope, true), + } +} + +fn held_entry( cache: &NodeStateCache, (location_id, target): (Uuid, &str), + watch: bool, ) -> Arc { let mut targets = cache.targets.lock().unwrap_or_else(PoisonError::into_inner); - targets + let entry = targets .entry((location_id, target.to_owned())) .or_insert_with(|| { Arc::new(TargetCache { target_hash: sha256_hex(target.as_bytes()), ..TargetCache::default() }) - }) - .clone() + }); + if watch { + entry.watchers.fetch_add(1, Ordering::SeqCst); + } + entry.clone() } pub(crate) fn held_view(entry: &TargetCache) -> Option> { @@ -100,7 +115,6 @@ pub(crate) async fn refresh_view( catalog, states: Arc::new(states), changed_build: changed_build.map(Arc::new), - nodes_body: Default::default(), states_body: Default::default(), }); *entry.view.write().unwrap_or_else(PoisonError::into_inner) = Some(view.clone()); diff --git a/crates/sqlbuild-orchestration/src/node_states/main/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/node_states/main/list_person_catalog_nodes.rs index 49ae93c8d9..960ee5cea3 100644 --- a/crates/sqlbuild-orchestration/src/node_states/main/list_person_catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/node_states/main/list_person_catalog_nodes.rs @@ -1,4 +1,4 @@ -//! A person lists the current nodes with view on all of them, served from the held view and tagged with its snapshot and version. +//! A person lists the current nodes with view on all of them, without their states, served from the held view and tagged with its snapshot. use uuid::Uuid; diff --git a/crates/sqlbuild-orchestration/src/node_states/main/start_node_states.rs b/crates/sqlbuild-orchestration/src/node_states/main/start_node_states.rs index 68b846be82..9fb567f619 100644 --- a/crates/sqlbuild-orchestration/src/node_states/main/start_node_states.rs +++ b/crates/sqlbuild-orchestration/src/node_states/main/start_node_states.rs @@ -1,5 +1,6 @@ -//! Committed change signals mark held views stale; a lost signal or a resync refreshes every held view and pushes only what changed, since clients refetch on `all`. +//! Committed change signals refresh the views change streams watch and drop the others, which the next read loads again; a lost signal or a resync covers every held view, and only what changed is pushed, since clients refetch on `all`. +use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex, PoisonError}; use tokio::sync::broadcast; @@ -43,11 +44,11 @@ pub(crate) fn start_node_states( target_hash, topic: ChangeTopic::NodeStatus | ChangeTopic::Snapshot, }) => ( - held_targets(&cache, Some((location_id, &target_hash))), + watched_targets(&cache, Some((location_id, &target_hash))), true, ), Ok(ChangeMessage::Resync) | Err(broadcast::error::RecvError::Lagged(_)) => { - (held_targets(&cache, None), false) + (watched_targets(&cache, None), false) } Ok(_) => continue, Err(broadcast::error::RecvError::Closed) => { @@ -69,19 +70,26 @@ pub(crate) fn start_node_states( (cache, task) } -/// The held targets a signal names, or every held target. -fn held_targets( +/// The watched targets a signal names, or every watched target; the unwatched ones it names are dropped once no refresh is pending or due, since one still pushes the old entry's versions. +fn watched_targets( cache: &NodeStateCache, scope: Option<(Uuid, &str)>, ) -> Vec<((Uuid, String), Arc)> { - let targets = cache.targets.lock().unwrap_or_else(PoisonError::into_inner); + let mut targets = cache.targets.lock().unwrap_or_else(PoisonError::into_inner); + let named = |(location_id, _): &(Uuid, String), entry: &TargetCache| { + scope.is_none_or(|(location, target_hash)| { + *location_id == location && entry.target_hash == target_hash + }) + }; + targets.retain(|key, entry| { + !named(key, entry) + || entry.watchers.load(Ordering::SeqCst) > 0 + || entry.dirty.load(Ordering::SeqCst) + || entry.scheduled.load(Ordering::SeqCst) + }); targets .iter() - .filter(|((location_id, _), entry)| { - scope.is_none_or(|(location, target_hash)| { - *location_id == location && entry.target_hash == target_hash - }) - }) - .map(|(scope, entry)| (scope.clone(), entry.clone())) + .filter(|(key, entry)| named(key, entry)) + .map(|(key, entry)| (key.clone(), entry.clone())) .collect() } diff --git a/crates/sqlbuild-orchestration/src/node_states/main/watch_node_states.rs b/crates/sqlbuild-orchestration/src/node_states/main/watch_node_states.rs index 1528e52edd..41f76feaad 100644 --- a/crates/sqlbuild-orchestration/src/node_states/main/watch_node_states.rs +++ b/crates/sqlbuild-orchestration/src/node_states/main/watch_node_states.rs @@ -1,17 +1,18 @@ -//! A change stream subscribes before the target's view loads, so no state change after its first refetch is missed. +//! A change stream subscribes before the target's view loads, so no state change after its first refetch is missed, and holds the target watched while it is open. use tokio::sync::broadcast; use uuid::Uuid; -use crate::node_states::_helpers::views::current_view; -use crate::node_states::models::{NodeStateCache, NodeStatusUpdate}; +use crate::node_states::_helpers::views::{current_view, watch_target}; +use crate::node_states::models::{NodeStateCache, NodeStatesWatch, NodeStatusUpdate}; use crate::store::errors::ControlError; pub(crate) async fn watch_node_states( cache: &NodeStateCache, scope: (Uuid, &str), -) -> Result, ControlError> { +) -> Result<(broadcast::Receiver, NodeStatesWatch), ControlError> { let updates = cache.updates.subscribe(); + let watch = watch_target(cache, scope); current_view(cache, scope, None).await?; - Ok(updates) + Ok((updates, watch)) } diff --git a/crates/sqlbuild-orchestration/src/node_states/models.rs b/crates/sqlbuild-orchestration/src/node_states/models.rs index aab7d14fd7..894e3d4924 100644 --- a/crates/sqlbuild-orchestration/src/node_states/models.rs +++ b/crates/sqlbuild-orchestration/src/node_states/models.rs @@ -1,7 +1,7 @@ //! Node states, their versioned in-memory views and the state changes pushed on the change stream. use std::collections::BTreeMap; -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; use axum::body::Bytes; @@ -98,16 +98,29 @@ pub(crate) struct TargetCache { pub(crate) scheduled: AtomicBool, /// A store signal named this target since its last refresh, so the refresh pushes a notice even when nothing changed. pub(crate) announce: AtomicBool, + /// Open change streams of this target; a signal refreshes a watched target and drops an unwatched one. + pub(crate) watchers: AtomicUsize, } -/// One version of a target's states; each response body is serialized at most once. +/// One change stream's hold on its target, which keeps the target's view refreshed until it drops. +#[derive(Debug)] +pub(crate) struct NodeStatesWatch { + pub(crate) entry: Arc, +} + +impl Drop for NodeStatesWatch { + fn drop(&mut self) { + self.entry.watchers.fetch_sub(1, Ordering::SeqCst); + } +} + +/// One version of a target's states; its states body is serialized at most once. #[derive(Debug)] pub(crate) struct TargetView { pub(crate) version: u64, pub(crate) catalog: Arc, pub(crate) states: Arc>, pub(crate) changed_build: Option>, - pub(crate) nodes_body: tokio::sync::OnceCell, pub(crate) states_body: tokio::sync::OnceCell, } @@ -117,6 +130,8 @@ pub(crate) struct SnapshotCatalog { pub(crate) snapshot_id: Option, pub(crate) records: Vec, pub(crate) inputs: Vec<(String, SnapshotNode)>, + /// The serialized records, which every state version of the snapshot lists. + pub(crate) body: tokio::sync::OnceCell, } /// One target's serialized `node_status` event data. diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_diff.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_diff.rs index e02ebe3ff2..4ff837c6b5 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_diff.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_diff.rs @@ -1,17 +1,16 @@ //! One planned model's diff text, under the plan's authorization, and only with view.sql. -use sqlbuild_core::plan_display::main::plan_model_diff::plan_model_diff; use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::platform::_helpers::plans::plan_access::read_person_plan_with_sql; use crate::platform::_helpers::plans::plan_diffs::read_plan_diff; -use crate::platform::models::{LocationWorkOutcome, PlanDiffRead, PlanModelDiff}; +use crate::platform::models::{LocationWorkOutcome, PlanDiffRead}; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; -/// The model's diff from the plan document or stored apart; `None` without one or without view.sql everywhere. +/// The model's diff the plan stored beside its document; `None` without one or without view.sql everywhere. pub async fn read_person_plan_diff( store: &ControlStore, reader: (&AuditContext, Uuid), @@ -20,24 +19,15 @@ pub async fn read_person_plan_diff( let Some((record, true)) = read_person_plan_with_sql(store, reader, plan).await? else { return Ok(None); }; - let Some(LocationWorkOutcome::Plan { document, .. }) = - record.result_json.map(|result| result.outcome) - else { + if !matches!( + record.result_json.map(|result| result.outcome), + Some(LocationWorkOutcome::Plan { .. }) + ) { return Ok(None); - }; - let inline = plan_model_diff(&document, model).map(|diff| PlanModelDiff { - query_diff: diff.query_diff, - config_diff: diff.config_diff, - }); - let diff = match inline { - Some(diff) => Some(diff), - None => { - let (transaction, stored) = - read_plan_diff(begin(store, TransactionMode::Read).await?, (plan.2, model)).await?; - commit(transaction).await?; - stored - } - }; + } + let (transaction, diff) = + read_plan_diff(begin(store, TransactionMode::Read).await?, (plan.2, model)).await?; + commit(transaction).await?; Ok(diff.map(|diff| PlanDiffRead { model: model.to_owned(), diff, diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 5e50a95ed3..4bee656326 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -273,7 +273,7 @@ pub enum LocationWorkOutcome { Plan { #[schema(value_type = Object)] document: serde_json::Value, - /// Model diff text the document left out to stay under its cap; stored apart, never read back here. + /// Model diff text, which the document never carries; stored apart, never read back here. #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] diffs: std::collections::BTreeMap, }, @@ -726,7 +726,7 @@ pub struct PlanDiffQuery { pub model: String, } -/// A planned model's diff text, from the plan document or, where the document left it out, apart from it. +/// A planned model's diff text, stored beside its plan document. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct PlanDiffRead { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs index 0e13f13e9a..97cd1902bf 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publications.rs @@ -102,16 +102,16 @@ pub(crate) async fn cached_publication( .await } -/// The value cached for `snapshot_id`, built once with `build` while concurrent callers wait; a failed build caches nothing. -pub(crate) async fn cached( - slots: &Mutex>, +/// The publication cached for `snapshot_id`, built once with `build` while concurrent callers wait; a failed build caches nothing. +pub(crate) async fn cached( + slots: &Mutex, key: ((Uuid, String), Uuid), state: S, build: F, -) -> Result<(S, Arc), ControlError> +) -> Result<(S, Arc), ControlError> where F: FnOnce(S) -> Fut, - Fut: Future>, + Fut: Future>, { let cell = slot(slots, key); let mut state = Some(state); @@ -132,10 +132,10 @@ where } /// The snapshot's cell, marked most recently used; adding one evicts its scope's least recently used beyond the limit. -fn slot( - slots: &Mutex>, +fn slot( + slots: &Mutex, (scope, snapshot_id): ((Uuid, String), Uuid), -) -> Arc>> { +) -> Arc>> { let mut state = slots.lock().unwrap_or_else(PoisonError::into_inner); state.clock += 1; let now = state.clock; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index c8571e0401..1d159c9386 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -927,7 +927,7 @@ impl Default for RunLogRetention { /// Prepared publications by snapshot ID; snapshots never change, so an entry stays valid until evicted. #[derive(Debug)] pub struct PublicationCache { - pub(crate) slots: std::sync::Mutex>, + pub(crate) slots: std::sync::Mutex, } impl PublicationCache { @@ -945,15 +945,15 @@ impl Default for PublicationCache { } } -/// Values built once per snapshot, least recently used first out within each location and target. +/// Publications prepared once per snapshot, least recently used first out within each location and target. #[derive(Debug)] -pub(crate) struct SnapshotSlots { +pub(crate) struct SnapshotSlots { /// Counts lookups, so a larger `used` is more recent. pub(crate) clock: u64, - pub(crate) slots: Vec>, + pub(crate) slots: Vec, } -impl SnapshotSlots { +impl SnapshotSlots { pub(crate) const fn new() -> Self { Self { clock: 0, @@ -962,13 +962,13 @@ impl SnapshotSlots { } } -/// One snapshot's value, built by the first request that needs it while later ones wait. +/// One snapshot's publication, prepared by the first request that needs it while later ones wait. #[derive(Debug)] -pub(crate) struct SnapshotSlot { +pub(crate) struct SnapshotSlot { pub(crate) scope: (Uuid, String), pub(crate) snapshot_id: Uuid, pub(crate) used: u64, - pub(crate) cell: std::sync::Arc>>, + pub(crate) cell: std::sync::Arc>>, } /// What every plan of one publication shares: the planner's index, each node's write-set facts and every node key. diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index f5691eadb0..78d596d4b2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -982,7 +982,7 @@ impl StateHistoryRetention { } } -/// A current snapshot node's declared metadata and latest build, without SQL, source or config. +/// A current snapshot node's declared metadata and latest build, which the node listing leaves to node-states. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogNodeRecord { pub node_key: String, diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index 30af0488af..936e14d9e5 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use sqlbuild_core::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::node_states::models::{NodeState, NodeStatesRecord}; use crate::runs::tests::helpers::{request, request_run, snapshot}; use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotInput; @@ -30,7 +31,7 @@ const LISTING_VIEW_STATEMENTS: usize = 6; const WIDE_LISTING_FIXED_STATEMENTS: usize = LISTING_AUTHORIZATION_STATEMENTS + LISTING_VIEW_STATEMENTS + 1; -/// A new server's node list, so every listing reads the store. +/// A new server's node list with each node's state read beside it, as a person sees it, so every listing reads the store. async fn list_person_catalog_nodes( store: &ControlStore, credentials: (&AuditContext, Uuid), @@ -48,7 +49,32 @@ async fn list_person_catalog_nodes( default_role, ) .await?; - Ok(serde_json::from_slice(&listing.body).expect("node list JSON")) + let states = crate::node_states::main::read_person_node_states::read_person_node_states( + &cache, + credentials, + scope, + default_role, + ) + .await?; + let mut states: std::collections::BTreeMap = + serde_json::from_slice::(&states.body) + .expect("node states JSON") + .states; + let records: Vec = + serde_json::from_slice(&listing.body).expect("node list JSON"); + Ok(records + .into_iter() + .map(|record| { + let state: NodeState = states.remove(&record.node_key).unwrap_or_default(); + CatalogNodeRecord { + last_build: state.last_build, + last_outcome: state.last_outcome, + definition_changed: state.definition_changed, + checks: state.checks, + ..record + } + }) + .collect()) } pub(crate) async fn node_status_reads( @@ -626,10 +652,14 @@ async fn list_wide_target( assert_eq!(built, size, "every wide node keeps its own build"); } timings.sort(); + let (cache, _) = crate::node_states::main::start_node_states::start_node_states( + store, + &tokio::sync::broadcast::channel(1).0, + ); let statements = STATEMENTS .scope(Cell::new(0), async { - list_person_catalog_nodes( - store, + crate::node_states::main::list_person_catalog_nodes::list_person_catalog_nodes( + &cache, credentials, (location, "development"), DefaultRole::None, diff --git a/crates/sqlbuild-orchestration/tests/api/test_checks.rs b/crates/sqlbuild-orchestration/tests/api/test_checks.rs index c52691ef12..c645d30808 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_checks.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_checks.rs @@ -102,22 +102,14 @@ fn given_checks_sharing_an_audit_name_when_people_read_them_then_binding_keys_an ], "one entry per binding key, each its newest result, marked when its build ignored it" ); - let (status, nodes) = get( + let (status, states) = get( &client, (&api.url, &viewer), - &format!("/api/control/locations/{location}/targets/development/nodes"), + &format!("/api/control/locations/{location}/targets/development/node-states"), ) .await; assert_eq!(status, StatusCode::OK); - let checks_of = |key: &str| { - nodes - .as_array() - .expect("nodes") - .iter() - .find(|node| node["node_key"] == key) - .map(|node| node["checks"].clone()) - .unwrap_or_else(|| panic!("{key} listed")) - }; + let checks_of = |key: &str| states["states"][key]["checks"].clone(); assert_eq!( checks_of("orders"), json!({"total": 1, "passing": 1, "warning": 0, "failing": 0}), diff --git a/crates/sqlbuild-orchestration/tests/api/test_node_states.rs b/crates/sqlbuild-orchestration/tests/api/test_node_states.rs index 78bcbcc971..e2c8a234ca 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_node_states.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_node_states.rs @@ -1,6 +1,7 @@ //! Held node lists and states: ETag revalidation, pushed state changes, recovery reads and snapshot resets on both control stores. use std::collections::BTreeSet; +use std::time::Duration; use reqwest::{StatusCode, header}; use serde_json::{Value, json}; @@ -11,10 +12,10 @@ use crate::api::tests::changes_helpers::{next_notice, opened, request_person_fee use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; use crate::api::tests::node_states_helpers::{builds, notices, orders_snapshot, reader}; use crate::api::tests::test_types::{ - ChangedBuildTestCase, NodeStatesTestCase, PushedStatesTestCase, + ChangedBuildTestCase, HeldTargetTestCase, NodeStatesTestCase, PushedStatesTestCase, }; use crate::credentials::tests::role_helpers::role_fixture; -use crate::node_states::_helpers::views::{current_view, refresh_view, target_cache}; +use crate::node_states::_helpers::views::{current_view, refresh_view, target_cache, watch_target}; use crate::node_states::main::report_changed_build::report_changed_build; use crate::node_states::main::start_node_states::start_node_states; use crate::node_states::models::ChangedBuildReport; @@ -23,9 +24,11 @@ use crate::runs::models::JobRunRequest; use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; +use crate::store::models::{ChangeMessage, ChangeTopic}; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::warehouse_metadata::main::ingest::ingest_fingerprints::ingest_fingerprints; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; #[test] fn given_held_node_states_when_people_revalidate_and_builds_land_then_only_changes_are_pushed_and_snapshots_reset() @@ -111,13 +114,13 @@ fn given_held_node_states_when_people_revalidate_and_builds_land_then_only_chang other_tag ), ( - format!("\"{}:1\"", published.snapshot_id), + format!("\"{}\"", published.snapshot_id), test_case.expected_listed, StatusCode::NOT_MODIFIED, first_tag.clone(), StatusCode::OK ), - "{}: the list is tagged with its snapshot and version and revalidates to 304", + "{}: the list is tagged with its snapshot and revalidates to 304", test_case.description ); assert_eq!( @@ -217,20 +220,11 @@ fn given_held_node_states_when_people_revalidate_and_builds_land_then_only_chang ); let relisted = read("nodes", &admin, Some(&first_tag)) .await - .expect("stale revalidation"); - let second_tag = etag(&relisted); - let relisted: Vec = relisted.json().await.expect("relisted"); - let orders = relisted - .iter() - .find(|node| node["node_key"] == "model:orders") - .expect("orders listed"); + .expect("revalidation after a build"); assert_eq!( - (second_tag, &orders["last_build"]), - ( - format!("\"{}:2\"", published.snapshot_id), - &pushed["states"]["model:orders"]["last_build"] - ), - "{}: a changed state invalidates the old ETag", + (relisted.status(), etag(&relisted)), + (StatusCode::NOT_MODIFIED, first_tag.clone()), + "{}: a changed state keeps the list's ETag, since node-states serves states", test_case.description ); @@ -254,7 +248,7 @@ fn given_held_node_states_when_people_revalidate_and_builds_land_then_only_chang json!({"topic": "node_status:development", "version": 3, "previous_version": 2, "full": true}), json!({"topic": format!("snapshot:{location}")}), ], - format!("\"{}:3\"", republished.snapshot_id), + format!("\"{}\"", republished.snapshot_id), ), "{}: a new snapshot resets the list and tells clients to refetch the states", test_case.description @@ -527,3 +521,85 @@ fn given_builds_of_many_nodes_when_states_refresh_then_large_changes_ask_clients }); } } + +#[test] +fn given_held_node_states_when_a_signal_names_their_target_then_only_watched_targets_stay_held() { + let test_cases = [ + HeldTargetTestCase { + description: "a target nobody watches is dropped and read again on the next request", + watched: false, + pending: (false, false), + expected_held_and_refreshed: (false, false), + }, + HeldTargetTestCase { + description: "a watched target stays held and refreshes", + watched: true, + pending: (false, false), + expected_held_and_refreshed: (true, true), + }, + HeldTargetTestCase { + description: "an unwatched target with a refresh pending stays held until it finishes", + watched: false, + pending: (false, true), + expected_held_and_refreshed: (true, false), + }, + HeldTargetTestCase { + description: "an unwatched target with another refresh pass due stays held and refreshes", + watched: false, + pending: (true, false), + expected_held_and_refreshed: (true, true), + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = role_fixture(store).await.location_id; + let mut input = orders_snapshot(location, &["orders"]); + input.generation = 2; + publish_snapshot(store, &input).await.expect("orders"); + let changes = tokio::sync::broadcast::channel(16).0; + let (cache, task) = start_node_states(store, &changes); + let task = tokio::spawn(task); + let scope = (location, "development"); + let watch = test_case.watched.then(|| watch_target(&cache, scope)); + current_view(&cache, scope, None).await.expect("first view"); + let entry = target_cache(&cache, scope); + let (dirty, scheduled) = test_case.pending; + entry + .dirty + .store(dirty, std::sync::atomic::Ordering::SeqCst); + entry + .scheduled + .store(scheduled, std::sync::atomic::Ordering::SeqCst); + let mut updates = cache.updates.subscribe(); + changes + .send(ChangeMessage::Changed { + location_id: location, + target_hash: sha256_hex(b"development"), + topic: ChangeTopic::NodeStatus, + }) + .expect("signal"); + let refreshed = tokio::time::timeout(Duration::from_secs(2), updates.recv()) + .await + .is_ok(); + let held = cache + .targets + .lock() + .expect("targets") + .contains_key(&(location, "development".to_owned())); + assert_eq!( + (held, refreshed), + test_case.expected_held_and_refreshed, + "{}", + test_case.description + ); + drop(watch); + task.abort(); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index 01dcfc07f3..0226a28950 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -560,29 +560,10 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an "config_diff": ["-a", "+b"]}), ]) .collect(); - let (detached_plan, detached) = - detach_model_diffs(json!({"models": models[..451].to_vec(), "warnings": []})); - let document = json!({ - "models": detached_plan["models"] - .as_array() - .into_iter() - .flatten() - .chain([&models[451]]) - .collect::>(), - "warnings": [], - }); - let diffs: BTreeMap = detached - .into_iter() - .map(|(model, diff)| { - ( - model, - PlanModelDiff { - query_diff: diff.query_diff, - config_diff: diff.config_diff, - }, - ) - }) - .collect(); + let (document, detached) = + detach_model_diffs(json!({"models": models, "warnings": []})); + let diffs: BTreeMap = + serde_json::from_value(json!(detached)).expect("diffs"); let result = LocationWorkResult { claim_id: claimed.claim_id, session_id: poll.session_id, @@ -629,7 +610,7 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an let detached_diff = read_person_plan_diff(store, reader, (plan, "orders_123")) .await .expect("diff"); - let inline_diff = read_person_plan_diff(store, reader, (plan, "customers")) + let config_diff = read_person_plan_diff(store, reader, (plan, "customers")) .await .expect("diff"); let missing = read_person_plan_diff(store, reader, (plan, "nowhere")) @@ -649,7 +630,7 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an ), (found.total, found.section_total), detached_diff.map(|read| read.diff.query_diff), - inline_diff.map(|read| read.diff.config_diff), + config_diff.map(|read| read.diff.config_diff), missing, stored.result_json.map(|result| matches!( result.outcome, @@ -674,7 +655,7 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an None, Some(true), ), - "{}: pages count the left-out diffs, each diff reads from the document or apart from it, \ + "{}: pages count the left-out diffs, each diff reads from where it was stored apart, \ and the stored plan never carries them", test_case.description ); @@ -693,7 +674,7 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an (json!("totals"), Some(json!("full_refresh true → false"))), (json!("customers"), None), ], - "{}: a short config change reads in words whether its diff was detached or not", + "{}: a short config change reads in words without its diff", test_case.description ); execute( diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 72c8c81ca1..dad657f254 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -358,6 +358,15 @@ pub(crate) struct ChangedBuildTestCase { pub(crate) expected_before_plan: serde_json::Value, } +pub(crate) struct HeldTargetTestCase { + pub(crate) description: &'static str, + pub(crate) watched: bool, + /// A refresh is due and one is scheduled, as when signals arrived while the target was watched. + pub(crate) pending: (bool, bool), + /// Whether the target is still held after the signal, and whether the signal refreshed it. + pub(crate) expected_held_and_refreshed: (bool, bool), +} + pub(crate) struct PushedStatesTestCase { pub(crate) description: &'static str, pub(crate) built: usize, diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 0118584bcd..3463250ef9 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -471,9 +471,9 @@ pub(crate) fn cache_id(number: u8) -> Uuid { Uuid::from_u128(u128::from(number) + 1) } -/// The cached value of `snapshot` in target `scope`, built as the snapshot's number, and whether this lookup built it. +/// The cached publication of `snapshot` in target `scope`, built as `marked(snapshot)`, its mark, and whether this lookup built it. pub(crate) async fn cache_lookup( - slots: &std::sync::Mutex>, + slots: &std::sync::Mutex, (scope, snapshot): (u8, u8), ) -> (u8, bool) { let built = std::cell::Cell::new(false); @@ -483,12 +483,32 @@ pub(crate) async fn cache_lookup( (), |()| async { built.set(true); - Ok(((), snapshot)) + Ok(((), marked(snapshot))) }, ) .await .expect("cached value"); - (*value, built.get()) + (mark(&value), built.get()) +} + +/// An empty publication whose only node key is `mark`, so a lookup shows which build it got. +pub(crate) fn marked(mark: u8) -> crate::runs::models::CachedPublication { + use crate::runs::types::CommandPlanner; + crate::runs::models::CachedPublication { + prepared: crate::runs::tests::test_types::ExactKeyPlanner + .prepare(Vec::new(), Default::default()), + writers: std::collections::BTreeMap::new(), + node_keys: std::iter::once(mark.to_string()).collect(), + } +} + +/// The mark `marked` gave a publication. +pub(crate) fn mark(publication: &crate::runs::models::CachedPublication) -> u8 { + publication + .node_keys + .first() + .and_then(|key| key.parse().ok()) + .expect("a marked publication") } /// A selection evaluation as `(count, page, next offset, refusal message)`. diff --git a/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs b/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs index 40c2637018..244a18b3ef 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_publication_cache.rs @@ -12,7 +12,7 @@ use crate::runs::_helpers::lifecycle::publications::cached; use crate::runs::main::requests::check_adhoc_run::check_adhoc_run; use crate::runs::models::{AdhocCheckRequest, SnapshotSlots}; use crate::runs::tests::helpers::{ - adhoc_fixture, cache_id, cache_lookup, check_projection, snapshot, + adhoc_fixture, cache_id, cache_lookup, check_projection, mark, marked, snapshot, }; use crate::runs::tests::test_types::{ CacheLookupTestCase, ConcurrentLookupTestCase, ExactKeyPlanner, FailedBuildTestCase, @@ -85,18 +85,18 @@ fn given_concurrent_first_lookups_when_one_is_building_then_the_others_wait_for_ }]; for test_case in test_cases { run(async { - let slots: Mutex> = Mutex::new(SnapshotSlots::new()); + let slots: Mutex = Mutex::new(SnapshotSlots::new()); let builds = Cell::new(0); let (release, released) = oneshot::channel::<()>(); let key = || ((cache_id(1), "development".to_owned()), cache_id(1)); let first = cached(&slots, key(), (), |()| async { builds.set(builds.get() + 1); released.await.expect("release the build"); - Ok(((), test_case.first_value)) + Ok(((), marked(test_case.first_value))) }); let second = cached(&slots, key(), (), |()| async { builds.set(builds.get() + 1); - Ok(((), test_case.second_value)) + Ok(((), marked(test_case.second_value))) }); let releasing = async { tokio::task::yield_now().await; @@ -106,7 +106,12 @@ fn given_concurrent_first_lookups_when_one_is_building_then_the_others_wait_for_ let ((), first) = first.expect("first lookup"); let ((), second) = second.expect("second lookup"); assert_eq!( - (*first, *second, builds.get(), Arc::ptr_eq(&first, &second)), + ( + mark(&first), + mark(&second), + builds.get(), + Arc::ptr_eq(&first, &second) + ), test_case.expected_outcome, "{}", test_case.description @@ -128,20 +133,20 @@ fn given_a_failed_build_when_looked_up_again_then_nothing_was_cached() { }]; for test_case in test_cases { run(async { - let slots: Mutex> = Mutex::new(SnapshotSlots::new()); + let slots: Mutex = Mutex::new(SnapshotSlots::new()); let key = || ((cache_id(1), "development".to_owned()), cache_id(1)); let failed = cached(&slots, key(), (), |()| async { - Err::<((), u8), _>(ControlError::Invalid(test_case.failure.into())) + Err::<((), _), _>(ControlError::Invalid(test_case.failure.into())) }) .await; let retried = cached(&slots, key(), (), |()| async { - Ok(((), test_case.retried_value)) + Ok(((), marked(test_case.retried_value))) }) .await; assert_eq!( ( failed.err().map(|error| error.to_string()), - retried.ok().map(|((), value)| *value) + retried.ok().map(|((), value)| mark(&value)) ), ( test_case.expected_outcome.0.map(str::to_owned), diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index eda383756c..6cff1f1831 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -189,6 +189,11 @@ export class ApiError extends Error { } } +/** A server or recording without the read, such as the static demo's without node states. */ +export function isUnsupported(error: unknown): boolean { + return error instanceof ApiError && (error.status === 404 || error.status === 405 || error.status === 501); +} + /** The server refused the read or no longer has what it names, so an earlier answer must not stay on screen. */ export function isRefusal(error: unknown): boolean { return error instanceof ApiError && (error.status === 403 || error.status === 404); diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index bbbb9f87c1..9f31a1b7ec 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -256,6 +256,25 @@ describe('node listing and states', () => { expect(a).toBe(b); }); + it('applies the node states to a read not given the held listing, which carries none', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + const path = new URL(request.url).pathname; + if (path.endsWith('/node-states')) { + return Response.json({ snapshot_id: record.snapshot_id, version: 3, states: { 'model:orders': { definition_changed: false, checks: record.checks, last_outcome: { at: '2026-01-02T00:00:00Z', status: 'failed' } } } }); + } + if (path.endsWith('/column-lineage')) { + return Response.json({ nodes: [{ node_key: 'model:orders', columns: [] }], edges: [{ source_key: 'model:orders', source_column: 'id', target_key: 'model:orders', target_column: 'order_id', transform: 'direct' }] }); + } + return Response.json([record], { headers: { etag: '"s1"' } }); + }) + ); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + const lineage = await client.getColumnLineage(ORDERS, 'model:orders'); + expect(lineage?.models.map((m) => m.status)).toEqual(['fail']); + }); + it('reads every node state with its version', async () => { const { client } = server(); const states = await client.listNodeStates(ORDERS); @@ -405,7 +424,8 @@ describe('overview', () => { 'fetch', vi.fn(async (request: Request) => { const path = new URL(request.url).pathname; - if (path.endsWith('/nodes') && nodesStatus !== 200) return Response.json({ error: 'refused' }, { status: nodesStatus }); + if ((path.endsWith('/nodes') || path.endsWith('/node-states')) && nodesStatus !== 200) return Response.json({ error: 'refused' }, { status: nodesStatus }); + if (path.endsWith('/node-states')) return Response.json({ snapshot_id: null, version: 1, states: {} }); if (path.endsWith('/platform/problems')) return Response.json({ scheduler_leader: 'server-1', scheduler_lease_expires_at: null, broken_locations: [], overdue_ticks: [] }); return Response.json([]); }) @@ -415,6 +435,8 @@ describe('overview', () => { it('is null for a person who may view only part of the target, with no partial counts', async () => { expect(await server(403).overview(ORDERS)).toBeNull(); + // The states are read only once the listing is allowed, so the refusal is recorded once. + expect(vi.mocked(fetch).mock.calls.filter(([request]) => new URL((request as Request).url).pathname.endsWith('/node-states'))).toHaveLength(0); }); it('still fails on any other error', async () => { @@ -712,6 +734,7 @@ describe('getCatalogNode', () => { ['/bootstrap', { login_mode: 'password', access: { name: 'vic', principal_id: 'p', actions: [], locations: [{ location_id: '0191f0aa-orders', name: 'Orders', target: 'development', actions }] } }], ['/nodes', [{ node_key: 'model:orders', kind: 'model', name: 'orders', materialization: 'table', tags: [], dependencies: [], snapshot_id: 's' }]], ['/checks', []], + ['/node-states', { snapshot_id: 's', version: 1, states: {} }], ['/sql', { node_key: 'model:orders', snapshot_id: 's', raw_sql: 'select 1', compiled_sql: 'select 1' }] ]; vi.stubGlobal( @@ -869,6 +892,7 @@ describe('getCatalogNode statistics', () => { return Response.json({ records: page, next_after: page.length === 100 && last ? { observed_at: last.record.observed_at, event_id: last.event_id } : null }); } if (url.pathname.endsWith('/node-status')) return Response.json(null); + if (url.pathname.endsWith('/node-states')) return Response.json({ snapshot_id: 's', version: 1, states: {} }); return Response.json([]); }) ); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 9b700e1400..aa3d3cedf5 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -2,7 +2,8 @@ import createFetch from 'openapi-fetch'; import { shortName } from '$lib/build-area/labels'; import { isActiveRun } from '$lib/status'; import type { components, paths } from '$control/schema.d.ts'; -import { ApiError, type ApiClient } from './client'; +import { ApiError, isUnsupported, type ApiClient } from './client'; +import { applyNodeStates } from './node-states'; import { checkDetailOf, toCheck, toCheckEvidence, toCheckSql } from './http-checks'; import { findNode, nodeDetailOf, toCatalogNodes, nodeJobsOf, toMicrobatchCoverage, toNodeRuns, toColumnLineage, toNodeStates, toNodeStatusDelta } from './http-catalog'; import { auditMatches, toAuditEntry } from './http-audit'; @@ -291,7 +292,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const errorRefused = new Set(); // Targets whose health the person may not read; each refusal is audited, so Connections asks once per page load. const healthRefused = new Set(); - // The node listing per location with the server's ETag; the server answers a repeat with 304 until the snapshot or states change. + // The node listing per location with the server's ETag; the server answers a repeat with 304 until the snapshot changes. const listings = new Map(); // Statistics samples per relation, oldest first, with the cursor after the newest; a refresh reads only newer samples. const samples = new Map(); @@ -803,7 +804,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, disableAccount: (name: string): Promise => changePerson(name, 'disable'), enableAccount: (name: string): Promise => changePerson(name, 'enable'), async getColumnLineage(scope: Scope, ref: string, listing?: CatalogNode[]): Promise { - const nodes = listing ?? (await implemented.listCatalogNodes(scope)); + const nodes = listing ?? (await listingWithStates(scope)); const node = findNode(nodes, ref); if (!node) return null; const { locationId, target } = parseLocationKey(scope.locationId); @@ -815,7 +816,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, return toColumnLineage(data, nodes); }, async getCatalogNode(scope: Scope, ref: string, listing?: CatalogNode[]): Promise { - const nodes = listing ?? (await implemented.listCatalogNodes(scope)); + const nodes = listing ?? (await listingWithStates(scope)); const node = findNode(nodes, ref); if (!node) return null; const { locationId, target } = parseLocationKey(scope.locationId); @@ -853,7 +854,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, const reads = Promise.all([ recentRuns(scope), implemented.listChecks(scope), - listing ?? implemented.listCatalogNodes(scope), + listing ?? listingWithStates(scope), implemented.listSnapshotChanges(scope, { limit: 100 }), implemented.listPlatformEvents(), acknowledgementsOf(scope) @@ -963,6 +964,16 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, return started; } + /** The listing with its node states, for reads not given the catalog feed's nodes; a recording without the states read lists them itself. The states are read only after the listing is allowed, so a partial viewer is refused once. */ + async function listingWithStates(scope: Scope): Promise { + const nodes = await implemented.listCatalogNodes(scope); + const states = await implemented.listNodeStates(scope).catch((reason: unknown) => { + if (isUnsupported(reason)) return null; + throw reason; + }); + return states ? applyNodeStates(nodes, states.states) : nodes; + } + /** Sends the held listing's ETag, so an unchanged listing comes back as a 304 and is not sent or converted again. */ async function catalogListing(scope: Scope): Promise { const { locationId, target } = parseLocationKey(scope.locationId); diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index ac21520d39..7574fcde23 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -9,7 +9,7 @@ import { to } from '$lib/paths'; import type { BuiltInRole } from './types'; export type { ApiClient } from './client'; -export { ApiError, isRefusal } from './client'; +export { ApiError, isRefusal, isUnsupported } from './client'; export * from './types'; export * from './signals'; export { mergePlanPage } from './http-runs'; diff --git a/ui/src/lib/state/catalog-feed.test.ts b/ui/src/lib/state/catalog-feed.test.ts index b45b16359d..b56dd2354d 100644 --- a/ui/src/lib/state/catalog-feed.test.ts +++ b/ui/src/lib/state/catalog-feed.test.ts @@ -169,6 +169,22 @@ describe('CatalogFeed', () => { expect(s.reads).toEqual({ listing: 2, states: 2 }); }); + it('keeps the held states applied when a reconnect lists the nodes before the states answer', async () => { + const s = server(); + s.set({ states: { 'model:orders': failed('r2') } }); + const { feed } = feedOver(s.client); + feed.ensure(SCOPE); + await settle(); + const open = s.hold(); + feed.handle(signal('all')); + await vi.advanceTimersByTimeAsync(400); + expect(s.reads.listing).toBe(2); + expect(feed.view.nodes?.find((n) => n.key === 'model:orders')?.lastOutcome).toBe('failed'); + open(); + await settle(); + expect(feed.view.nodes?.find((n) => n.key === 'model:orders')?.lastOutcome).toBe('failed'); + }); + it('holds changes that arrive while the states are read, and applies the ones after the answer', async () => { const s = server(); const open = s.hold(); diff --git a/ui/src/lib/state/catalog-feed.ts b/ui/src/lib/state/catalog-feed.ts index 1a6b6a9a75..8dc228affe 100644 --- a/ui/src/lib/state/catalog-feed.ts +++ b/ui/src/lib/state/catalog-feed.ts @@ -1,5 +1,5 @@ import type { ApiClient, CatalogNode, ChangedBuild, ChangedSignal, NodeState, NodeStatusDelta, Scope } from '$lib/api'; -import { ApiError, isRefusal } from '$lib/api'; +import { isRefusal, isUnsupported } from '$lib/api'; import { applyNodeStates, deltaAction } from '$lib/api/node-states'; /** How long a burst of signals may last before one read starts. */ @@ -10,9 +10,6 @@ export type CatalogView = { nodes: CatalogNode[] | undefined; error: string | nu const message = (reason: unknown) => (reason instanceof Error ? reason.message : String(reason)); -/** A server or recording without the node-states read; the listing then carries the states itself. */ -const unsupported = (reason: unknown) => reason instanceof ApiError && (reason.status === 404 || reason.status === 405 || reason.status === 501); - /** * The selected location's catalog nodes, shared by every page. The listing is read once per * snapshot; node states are read once with their version, and `node_status` changes are applied @@ -21,8 +18,9 @@ const unsupported = (reason: unknown) => reason instanceof ApiError && (reason.s * nothing changed. `runs` and `checks` signals read nothing: their effect on nodes arrives as a * `node_status` change. * - * Against a server without the node-states read, or one whose `node_status` signals carry no - * states, a `node_status` signal reads the listing again, as pages did before. + * The listing carries no states, so the held states stay applied while both are read again. Against + * a recording without the node-states read, whose listing carries the states, or signals that carry + * no states, a `node_status` signal reads the listing again. */ export class CatalogFeed { #view: CatalogView = { nodes: undefined, error: null, loading: false, changedBuild: null }; @@ -118,8 +116,6 @@ export class CatalogFeed { if (!scope) return; const epoch = this.#epoch; if (states && !this.#statesUnsupported && this.client().supports('listNodeStates')) { - // Both read together: the listing's own states are at least as new as the overlay was. - if (listing) this.#overlay = {}; this.#readStates(scope, epoch); } else if (states) this.#statesUnsupported = true; if (listing) this.#readListing(scope, epoch); @@ -173,7 +169,7 @@ export class CatalogFeed { this.#version = null; this.#queued = []; // The listing carries the states, so from now on a change reads it again. - if (unsupported(reason)) this.#statesUnsupported = true; + if (isUnsupported(reason)) this.#statesUnsupported = true; } ); }