Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1650,7 +1650,7 @@ export interface components {
run_id?: string | null;
status: string;
};
/** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */
/** @description A current snapshot node's declared metadata and latest build, which the node listing leaves to node-states. */
CatalogNodeRecord: {
/** @description The checks on the node by their newest result. */
checks: components["schemas"]["CatalogNodeChecks"];
Expand Down Expand Up @@ -2275,7 +2275,7 @@ export interface components {
kind: "sensor_evaluation";
outcome: components["schemas"]["CustomSensorOutcome"];
} | {
/** @description Model diff text the document left out to stay under its cap; stored apart, never read back here. */
/** @description Model diff text, which the document never carries; stored apart, never read back here. */
diffs?: {
[key: string]: components["schemas"]["PlanModelDiff"];
};
Expand Down Expand Up @@ -2739,7 +2739,7 @@ export interface components {
/** @enum {string} */
status: "unavailable";
};
/** @description A planned model's diff text, from the plan document or, where the document left it out, apart from it. */
/** @description A planned model's diff text, stored beside its plan document. */
PlanDiffRead: components["schemas"]["PlanModelDiff"] & {
model: string;
};
Expand Down Expand Up @@ -5643,7 +5643,7 @@ export interface operations {
parameters: {
query?: never;
header?: {
/** @description An ETag from an earlier response; 304 when the snapshot and state version are unchanged */
/** @description An ETag from an earlier response; 304 when the snapshot is unchanged */
"If-None-Match"?: string | null;
};
path: {
Expand All @@ -5654,10 +5654,10 @@ export interface operations {
};
requestBody?: never;
responses: {
/** @description Every node of the current snapshot; empty before the first publication */
/** @description Every node of the current snapshot without its state, which node-states serves; empty before the first publication */
200: {
headers: {
/** @description "<snapshot_id>:<state_version>"; absent before the first publication */
/** @description "<snapshot_id>"; absent before the first publication */
ETag?: string;
[name: string]: unknown;
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use std::collections::BTreeMap;

use serde_json::Value;
use serde_json::{Value, json};
use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs;
use sqlbuild_orchestration::platform::constants::{PLAN_DIFF_BYTES, PLAN_DOCUMENT_BYTES};
use sqlbuild_orchestration::platform::models::PlanModelDiff;
Expand All @@ -11,7 +11,7 @@ use crate::native_commands::_helpers::location::execution::{plan_failure, withou
use crate::native_commands::constants::{PLAN_NOT_PRODUCED, PLAN_OUTPUT_BYTES};
use crate::native_commands::types::LocationPlan;

/// The plan `sqb plan --json` printed; over the cap its model diffs leave it, keeping their counts, and travel beside it.
/// The plan `sqb plan --json` printed; its model diffs leave it, keeping their counts, and travel beside it.
pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) -> LocationPlan {
if output.len() > PLAN_OUTPUT_BYTES {
return without_diffs(plan_failure("the plan is too large to show"));
Expand All @@ -20,28 +20,16 @@ pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) ->
Ok(document) if document.is_object() && planned => document,
_ => return without_diffs(plan_failure(PLAN_NOT_PRODUCED)),
};
if output.len() <= PLAN_DOCUMENT_BYTES {
return without_diffs(document);
}
let (document, detached) = detach_model_diffs(document);
if !serde_json::to_vec(&document).is_ok_and(|bytes| bytes.len() <= PLAN_DOCUMENT_BYTES) {
return without_diffs(plan_failure("the plan is too large to show"));
}
let diffs: BTreeMap<String, PlanModelDiff> = detached
.into_iter()
.map(|(model, diff)| {
(
model,
PlanModelDiff {
query_diff: diff.query_diff,
config_diff: diff.config_diff,
},
)
})
.collect();
if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) {
(document, diffs)
} else {
without_diffs(document)
match serde_json::from_value::<BTreeMap<String, PlanModelDiff>>(json!(detached)) {
Ok(diffs)
if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) =>
{
(document, diffs)
}
_ => without_diffs(document),
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,20 @@ use crate::native_commands::tests::helpers::plan_output;
use crate::native_commands::tests::test_types::{PlanDocumentTestCase, WorkCheckTestCase};

#[test]
fn given_plan_output_when_reading_it_on_a_location_then_only_oversized_diffs_leave_the_document() {
fn given_plan_output_when_reading_it_on_a_location_then_diffs_always_travel_beside_the_document() {
let test_cases = [
PlanDocumentTestCase {
description: "a plan under the cap keeps its diffs",
description: "a small plan's diffs leave the document with their counts too",
output: plan_output("+SELECT 2", 2),
planned: true,
expected_first_model: json!({"name": "model_0", "reason": "query_changed", "query_diff": ["+SELECT 2"], "config_diff": ["+a"]}),
expected_diffs: (0, None),
expected_first_model: json!({"name": "model_0", "reason": "query_changed", "diff_stats": {
"query": {"added": 1, "removed": 0, "whitespace_only": false},
"config": {"added": 1, "removed": 0, "whitespace_only": false},
}}),
expected_diffs: (2, Some(vec!["+SELECT 2".to_owned()])),
},
PlanDocumentTestCase {
description: "diffs over the cap leave the document with their counts and travel beside it",
description: "a plan over the document cap keeps every diff beside it",
output: plan_output(
&format!("+{}", "x".repeat(1024)),
PLAN_DOCUMENT_BYTES / 1024 + 1,
Expand Down
27 changes: 15 additions & 12 deletions crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,31 @@

use std::collections::BTreeMap;

use serde_json::Value;
use serde_json::{Map, Value};

use crate::plan_display::_helpers::diff_lines::config_change;
use crate::plan_display::_helpers::plan_pages::{lines, row_diff_stats, text};
use crate::plan_display::constants::{CONFIG_CHANGE_FIELD, DIFF_STATS_FIELD, MODEL_DIFF_FIELDS};
use crate::plan_display::models::PlanModelDiff;

/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and the text by model.
pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap<String, PlanModelDiff>) {
let mut diffs: BTreeMap<String, PlanModelDiff> = BTreeMap::new();
/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and by model the `query_diff` and `config_diff` fields taken out.
pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap<String, Value>) {
let mut diffs: BTreeMap<String, Value> = BTreeMap::new();
let models = document.get_mut("models").and_then(Value::as_array_mut);
for model in models.into_iter().flatten() {
let diff = PlanModelDiff {
query_diff: lines(model, "query_diff"),
config_diff: lines(model, "config_diff"),
};
if diff.query_diff.is_none() && diff.config_diff.is_none() {
let mut diff: Map<String, Value> = Map::new();
for field in MODEL_DIFF_FIELDS {
if let Some(text) = lines(model, field) {
diff.insert(field.to_owned(), Value::from(text));
}
}
if diff.is_empty() {
continue;
}
let stats = row_diff_stats(model);
let change = diff.config_diff.as_deref().and_then(config_change);
diffs.insert(text(model, "name").to_owned(), diff);
let change = lines(model, "config_diff")
.as_deref()
.and_then(config_change);
diffs.insert(text(model, "name").to_owned(), Value::Object(diff));
if let Some(fields) = model.as_object_mut() {
for field in MODEL_DIFF_FIELDS {
fields.remove(field);
Expand Down
1 change: 0 additions & 1 deletion crates/sqlbuild-core/src/plan_display/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,5 @@ pub mod model_execution_annotation;
pub mod model_materialization_label;
pub mod model_resource_type;
pub mod old_name_rows;
pub mod plan_model_diff;
pub mod plan_section_page;
pub mod plan_summary;
20 changes: 0 additions & 20 deletions crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs

This file was deleted.

7 changes: 0 additions & 7 deletions crates/sqlbuild-core/src/plan_display/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,3 @@ pub struct DiffStats {
/// The removed and added lines read the same with every whitespace character taken out.
pub whitespace_only: bool,
}

/// A planned model's diff text, as `sqb plan` prints it.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct PlanModelDiff {
pub query_diff: Option<Vec<String>>,
pub config_diff: Option<Vec<String>>,
}
19 changes: 13 additions & 6 deletions crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ use serde_json::{Value, json};

use crate::plan_display::_helpers::diff_lines::diff_stats;
use crate::plan_display::main::detach_model_diffs::detach_model_diffs;
use crate::plan_display::main::plan_model_diff::plan_model_diff;
use crate::plan_display::main::plan_section_page::plan_section_page;
use crate::plan_display::main::plan_summary::plan_summary;
use crate::plan_display::models::{DiffStats, PlanPageSection};
Expand Down Expand Up @@ -591,20 +590,28 @@ fn given_a_document_whose_diffs_were_detached_when_paged_then_counts_remain_and_
let page = |document: &Value| {
plan_section_page(document, PlanPageSection::QueryChanged, ((0, 50), None)).rows
};
let model_diff = |document: &Value| {
document["models"]
.as_array()
.into_iter()
.flatten()
.find(|row| row["name"] == test_case.model)
.and_then(|row| row.get("query_diff").cloned())
};
assert_eq!(
(
page(&detached),
diffs
.get(test_case.model)
.and_then(|diff| diff.query_diff.clone()),
plan_model_diff(&detached, test_case.model),
plan_model_diff(&original, test_case.model).and_then(|diff| diff.query_diff),
.and_then(|diff| diff.get("query_diff").cloned()),
model_diff(&detached),
model_diff(&original),
),
(
page(&original),
test_case.expected_detached.map(owned),
test_case.expected_detached.map(|lines| json!(lines)),
None,
test_case.expected_detached.map(owned),
test_case.expected_detached.map(|lines| json!(lines)),
),
"{}: pages read the same counts, and the text leaves the document whole",
test_case.description
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ async fn stream_changes(
(location_id, target): (Uuid, String),
) -> Result<Response, ApiError> {
let receiver = feed.sender.subscribe();
let updates = watch_node_states(&cache, (location_id, &target)).await?;
let (updates, watch) = watch_node_states(&cache, (location_id, &target)).await?;
let state = ChangeStream {
store,
subscriber,
Expand All @@ -119,6 +119,7 @@ async fn stream_changes(
target,
receiver,
updates,
_watch: watch,
stopped: feed.stopped,
keepalive: tokio::time::interval(Duration::from_secs(15)),
initial: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,9 +84,9 @@ pub(crate) async fn run_nodes(

#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes", operation_id = "listPersonCatalogNodes",
params(("location_id" = Uuid, Path), ("target" = String, Path),
("If-None-Match" = Option<String>, Header, description = "An ETag from an earlier response; 304 when the snapshot and state version are unchanged")),
responses((status = 200, body = Vec<CatalogNodeRecord>, description = "Every node of the current snapshot; empty before the first publication",
headers(("ETag" = String, description = "\"<snapshot_id>:<state_version>\"; absent before the first publication"))),
("If-None-Match" = Option<String>, Header, description = "An ETag from an earlier response; 304 when the snapshot is unchanged")),
responses((status = 200, body = Vec<CatalogNodeRecord>, description = "Every node of the current snapshot without its state, which node-states serves; empty before the first publication",
headers(("ETag" = String, description = "\"<snapshot_id>\"; absent before the first publication"))),
(status = 304, description = "Unchanged since the If-None-Match ETag", headers(("ETag" = String))),
(status = 401, description = "Session rejected"),
(status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))]
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-orchestration/src/api/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -575,6 +575,8 @@ pub(crate) struct ChangeStream {
pub(crate) receiver: tokio::sync::broadcast::Receiver<crate::store::models::ChangeMessage>,
pub(crate) updates:
tokio::sync::broadcast::Receiver<crate::node_states::models::NodeStatusUpdate>,
/// Keeps the target's node states refreshed while the stream is open.
pub(crate) _watch: crate::node_states::models::NodeStatesWatch,
pub(crate) stopped: tokio::sync::watch::Receiver<bool>,
pub(crate) keepalive: tokio::time::Interval,
pub(crate) initial: bool,
Expand Down
40 changes: 12 additions & 28 deletions crates/sqlbuild-orchestration/src/node_states/_helpers/bodies.rs
Original file line number Diff line number Diff line change
@@ -1,68 +1,52 @@
//! Response bodies are serialized once per view, off the async workers, and tagged with the view's snapshot and version.
//! Response bodies are serialized once, off the async workers: the node list once per snapshot, which also tags it, and the states once per view.

use std::sync::Arc;

use axum::body::Bytes;

use crate::node_states::models::{CachedJson, NodeStatesRecord, TargetView};
use crate::store::errors::ControlError;
use crate::warehouse_metadata::models::CatalogNodeRecord;

/// The view's node list with each node's state; an empty list before the first publication.
/// The snapshot's node list without states; an empty list before the first publication.
pub(crate) async fn nodes_json(view: &Arc<TargetView>) -> Result<CachedJson, ControlError> {
let catalog = view.catalog.clone();
let body = view
.nodes_body
.get_or_try_init(|| serialized(view.clone(), node_records))
.catalog
.body
.get_or_try_init(|| serialized(move || serde_json::to_vec(&catalog.records)))
.await?
.clone();
Ok(CachedJson {
etag: view
.catalog
.snapshot_id
.map(|snapshot_id| format!("\"{snapshot_id}:{}\"", view.version)),
.map(|snapshot_id| format!("\"{snapshot_id}\"")),
body,
})
}

/// The view's states with its snapshot and version.
pub(crate) async fn states_json(view: &Arc<TargetView>) -> Result<CachedJson, ControlError> {
let held = view.clone();
let body = view
.states_body
.get_or_try_init(|| serialized(view.clone(), state_record))
.get_or_try_init(|| serialized(move || serde_json::to_vec(&state_record(&held))))
.await?
.clone();
Ok(CachedJson { etag: None, body })
}

async fn serialized<T: serde::Serialize + 'static>(
view: Arc<TargetView>,
record: fn(&TargetView) -> T,
async fn serialized(
write: impl FnOnce() -> serde_json::Result<Vec<u8>> + Send + 'static,
) -> Result<Bytes, ControlError> {
tokio::task::spawn_blocking(move || {
serde_json::to_vec(&record(&view))
write()
.map(Bytes::from)
.map_err(|error| ControlError::Invalid(error.to_string()))
})
.await?
}

fn node_records(view: &TargetView) -> Vec<CatalogNodeRecord> {
view.catalog
.records
.iter()
.map(|record| {
let mut record = record.clone();
if let Some(state) = view.states.get(&record.node_key) {
record.last_build.clone_from(&state.last_build);
record.last_outcome.clone_from(&state.last_outcome);
record.definition_changed = state.definition_changed;
record.checks.clone_from(&state.checks);
}
record
})
.collect()
}

fn state_record(view: &TargetView) -> NodeStatesRecord {
NodeStatesRecord {
snapshot_id: view.catalog.snapshot_id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ fn snapshot_catalog(snapshot_id: Uuid, nodes: BTreeMap<String, SnapshotNode>) ->
snapshot_id: Some(snapshot_id),
records,
inputs,
body: Default::default(),
}
}

Expand Down
Loading
Loading