Skip to content

Commit 8816dfa

Browse files
authored
refactor(api): keep plan diffs in one place and drop unwatched node-state views (#1588)
* refactor(plan): keep every model diff beside the plan document A location always detaches model diff text and the server reads a diff only from where it stored it, so the inline lookup and the copied diff type go. * refactor(api): drop unwatched node-state views and tag the node list by snapshot The node states cache kept refreshing every target anyone had read for the life of the server. A change stream now holds its target watched while it is open; a store signal refreshes watched targets and drops the others, which the next read loads again. The node list no longer embeds states, which node-states serves, so it is serialized once per snapshot and its ETag is the snapshot ID: a build no longer invalidates it. * refactor(runs): make the publication cache concrete SnapshotSlots and cached only ever held prepared publications, so they name CachedPublication instead of a type parameter. * fix(ui): read node states beside the stateless node list The node list no longer carries states, so the catalog keeps the states it holds while the list and states are read again, and a node page or overview opened without the held list reads the states with it. The server keeps a dropped target's entry while a refresh is still pending, so the refresh cannot push the old entry's versions to a new stream. * fix(api): keep a node-states entry with a refresh due, and refuse a partial overview once A refresh that finds its target changed again runs another pass, so the entry stays held while one is due as well as while one is scheduled. A read without the held node list asks for the states only once the list is allowed, so a partial viewer's refusal is recorded once. * test(api): read node states beside the node list in catalog and check tests The node list no longer carries states, so the catalog-read and check-count tests read them from node-states as clients do. The wide-listing statement guard still counts the node list alone, within its unchanged budget.
1 parent 069c955 commit 8816dfa

36 files changed

Lines changed: 419 additions & 265 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1650,7 +1650,7 @@ export interface components {
16501650
run_id?: string | null;
16511651
status: string;
16521652
};
1653-
/** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */
1653+
/** @description A current snapshot node's declared metadata and latest build, which the node listing leaves to node-states. */
16541654
CatalogNodeRecord: {
16551655
/** @description The checks on the node by their newest result. */
16561656
checks: components["schemas"]["CatalogNodeChecks"];
@@ -2275,7 +2275,7 @@ export interface components {
22752275
kind: "sensor_evaluation";
22762276
outcome: components["schemas"]["CustomSensorOutcome"];
22772277
} | {
2278-
/** @description Model diff text the document left out to stay under its cap; stored apart, never read back here. */
2278+
/** @description Model diff text, which the document never carries; stored apart, never read back here. */
22792279
diffs?: {
22802280
[key: string]: components["schemas"]["PlanModelDiff"];
22812281
};
@@ -2739,7 +2739,7 @@ export interface components {
27392739
/** @enum {string} */
27402740
status: "unavailable";
27412741
};
2742-
/** @description A planned model's diff text, from the plan document or, where the document left it out, apart from it. */
2742+
/** @description A planned model's diff text, stored beside its plan document. */
27432743
PlanDiffRead: components["schemas"]["PlanModelDiff"] & {
27442744
model: string;
27452745
};
@@ -5643,7 +5643,7 @@ export interface operations {
56435643
parameters: {
56445644
query?: never;
56455645
header?: {
5646-
/** @description An ETag from an earlier response; 304 when the snapshot and state version are unchanged */
5646+
/** @description An ETag from an earlier response; 304 when the snapshot is unchanged */
56475647
"If-None-Match"?: string | null;
56485648
};
56495649
path: {
@@ -5654,10 +5654,10 @@ export interface operations {
56545654
};
56555655
requestBody?: never;
56565656
responses: {
5657-
/** @description Every node of the current snapshot; empty before the first publication */
5657+
/** @description Every node of the current snapshot without its state, which node-states serves; empty before the first publication */
56585658
200: {
56595659
headers: {
5660-
/** @description "<snapshot_id>:<state_version>"; absent before the first publication */
5660+
/** @description "<snapshot_id>"; absent before the first publication */
56615661
ETag?: string;
56625662
[name: string]: unknown;
56635663
};

‎crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_document.rs‎

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
33
use std::collections::BTreeMap;
44

5-
use serde_json::Value;
5+
use serde_json::{Value, json};
66
use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs;
77
use sqlbuild_orchestration::platform::constants::{PLAN_DIFF_BYTES, PLAN_DOCUMENT_BYTES};
88
use sqlbuild_orchestration::platform::models::PlanModelDiff;
@@ -11,7 +11,7 @@ use crate::native_commands::_helpers::location::execution::{plan_failure, withou
1111
use crate::native_commands::constants::{PLAN_NOT_PRODUCED, PLAN_OUTPUT_BYTES};
1212
use crate::native_commands::types::LocationPlan;
1313

14-
/// The plan `sqb plan --json` printed; over the cap its model diffs leave it, keeping their counts, and travel beside it.
14+
/// The plan `sqb plan --json` printed; its model diffs leave it, keeping their counts, and travel beside it.
1515
pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) -> LocationPlan {
1616
if output.len() > PLAN_OUTPUT_BYTES {
1717
return without_diffs(plan_failure("the plan is too large to show"));
@@ -20,28 +20,16 @@ pub(in crate::native_commands) fn plan_document(output: &[u8], planned: bool) ->
2020
Ok(document) if document.is_object() && planned => document,
2121
_ => return without_diffs(plan_failure(PLAN_NOT_PRODUCED)),
2222
};
23-
if output.len() <= PLAN_DOCUMENT_BYTES {
24-
return without_diffs(document);
25-
}
2623
let (document, detached) = detach_model_diffs(document);
2724
if !serde_json::to_vec(&document).is_ok_and(|bytes| bytes.len() <= PLAN_DOCUMENT_BYTES) {
2825
return without_diffs(plan_failure("the plan is too large to show"));
2926
}
30-
let diffs: BTreeMap<String, PlanModelDiff> = detached
31-
.into_iter()
32-
.map(|(model, diff)| {
33-
(
34-
model,
35-
PlanModelDiff {
36-
query_diff: diff.query_diff,
37-
config_diff: diff.config_diff,
38-
},
39-
)
40-
})
41-
.collect();
42-
if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) {
43-
(document, diffs)
44-
} else {
45-
without_diffs(document)
27+
match serde_json::from_value::<BTreeMap<String, PlanModelDiff>>(json!(detached)) {
28+
Ok(diffs)
29+
if serde_json::to_vec(&diffs).is_ok_and(|bytes| bytes.len() <= PLAN_DIFF_BYTES) =>
30+
{
31+
(document, diffs)
32+
}
33+
_ => without_diffs(document),
4634
}
4735
}

‎crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,20 @@ use crate::native_commands::tests::helpers::plan_output;
99
use crate::native_commands::tests::test_types::{PlanDocumentTestCase, WorkCheckTestCase};
1010

1111
#[test]
12-
fn given_plan_output_when_reading_it_on_a_location_then_only_oversized_diffs_leave_the_document() {
12+
fn given_plan_output_when_reading_it_on_a_location_then_diffs_always_travel_beside_the_document() {
1313
let test_cases = [
1414
PlanDocumentTestCase {
15-
description: "a plan under the cap keeps its diffs",
15+
description: "a small plan's diffs leave the document with their counts too",
1616
output: plan_output("+SELECT 2", 2),
1717
planned: true,
18-
expected_first_model: json!({"name": "model_0", "reason": "query_changed", "query_diff": ["+SELECT 2"], "config_diff": ["+a"]}),
19-
expected_diffs: (0, None),
18+
expected_first_model: json!({"name": "model_0", "reason": "query_changed", "diff_stats": {
19+
"query": {"added": 1, "removed": 0, "whitespace_only": false},
20+
"config": {"added": 1, "removed": 0, "whitespace_only": false},
21+
}}),
22+
expected_diffs: (2, Some(vec!["+SELECT 2".to_owned()])),
2023
},
2124
PlanDocumentTestCase {
22-
description: "diffs over the cap leave the document with their counts and travel beside it",
25+
description: "a plan over the document cap keeps every diff beside it",
2326
output: plan_output(
2427
&format!("+{}", "x".repeat(1024)),
2528
PLAN_DOCUMENT_BYTES / 1024 + 1,

‎crates/sqlbuild-core/src/plan_display/main/detach_model_diffs.rs‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,31 @@
22
33
use std::collections::BTreeMap;
44

5-
use serde_json::Value;
5+
use serde_json::{Map, Value};
66

77
use crate::plan_display::_helpers::diff_lines::config_change;
88
use crate::plan_display::_helpers::plan_pages::{lines, row_diff_stats, text};
99
use crate::plan_display::constants::{CONFIG_CHANGE_FIELD, DIFF_STATS_FIELD, MODEL_DIFF_FIELDS};
10-
use crate::plan_display::models::PlanModelDiff;
1110

12-
/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and the text by model.
13-
pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap<String, PlanModelDiff>) {
14-
let mut diffs: BTreeMap<String, PlanModelDiff> = BTreeMap::new();
11+
/// The document without model diff text, each model keeping its `diff_stats` and short `config_change`, and by model the `query_diff` and `config_diff` fields taken out.
12+
pub fn detach_model_diffs(mut document: Value) -> (Value, BTreeMap<String, Value>) {
13+
let mut diffs: BTreeMap<String, Value> = BTreeMap::new();
1514
let models = document.get_mut("models").and_then(Value::as_array_mut);
1615
for model in models.into_iter().flatten() {
17-
let diff = PlanModelDiff {
18-
query_diff: lines(model, "query_diff"),
19-
config_diff: lines(model, "config_diff"),
20-
};
21-
if diff.query_diff.is_none() && diff.config_diff.is_none() {
16+
let mut diff: Map<String, Value> = Map::new();
17+
for field in MODEL_DIFF_FIELDS {
18+
if let Some(text) = lines(model, field) {
19+
diff.insert(field.to_owned(), Value::from(text));
20+
}
21+
}
22+
if diff.is_empty() {
2223
continue;
2324
}
2425
let stats = row_diff_stats(model);
25-
let change = diff.config_diff.as_deref().and_then(config_change);
26-
diffs.insert(text(model, "name").to_owned(), diff);
26+
let change = lines(model, "config_diff")
27+
.as_deref()
28+
.and_then(config_change);
29+
diffs.insert(text(model, "name").to_owned(), Value::Object(diff));
2730
if let Some(fields) = model.as_object_mut() {
2831
for field in MODEL_DIFF_FIELDS {
2932
fields.remove(field);

‎crates/sqlbuild-core/src/plan_display/main/mod.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,5 @@ pub mod model_execution_annotation;
1111
pub mod model_materialization_label;
1212
pub mod model_resource_type;
1313
pub mod old_name_rows;
14-
pub mod plan_model_diff;
1514
pub mod plan_section_page;
1615
pub mod plan_summary;

‎crates/sqlbuild-core/src/plan_display/main/plan_model_diff.rs‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎crates/sqlbuild-core/src/plan_display/models.rs‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,10 +94,3 @@ pub struct DiffStats {
9494
/// The removed and added lines read the same with every whitespace character taken out.
9595
pub whitespace_only: bool,
9696
}
97-
98-
/// A planned model's diff text, as `sqb plan` prints it.
99-
#[derive(Clone, Debug, Default, PartialEq, Eq)]
100-
pub struct PlanModelDiff {
101-
pub query_diff: Option<Vec<String>>,
102-
pub config_diff: Option<Vec<String>>,
103-
}

‎crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ use serde_json::{Value, json};
22

33
use crate::plan_display::_helpers::diff_lines::diff_stats;
44
use crate::plan_display::main::detach_model_diffs::detach_model_diffs;
5-
use crate::plan_display::main::plan_model_diff::plan_model_diff;
65
use crate::plan_display::main::plan_section_page::plan_section_page;
76
use crate::plan_display::main::plan_summary::plan_summary;
87
use crate::plan_display::models::{DiffStats, PlanPageSection};
@@ -591,20 +590,28 @@ fn given_a_document_whose_diffs_were_detached_when_paged_then_counts_remain_and_
591590
let page = |document: &Value| {
592591
plan_section_page(document, PlanPageSection::QueryChanged, ((0, 50), None)).rows
593592
};
593+
let model_diff = |document: &Value| {
594+
document["models"]
595+
.as_array()
596+
.into_iter()
597+
.flatten()
598+
.find(|row| row["name"] == test_case.model)
599+
.and_then(|row| row.get("query_diff").cloned())
600+
};
594601
assert_eq!(
595602
(
596603
page(&detached),
597604
diffs
598605
.get(test_case.model)
599-
.and_then(|diff| diff.query_diff.clone()),
600-
plan_model_diff(&detached, test_case.model),
601-
plan_model_diff(&original, test_case.model).and_then(|diff| diff.query_diff),
606+
.and_then(|diff| diff.get("query_diff").cloned()),
607+
model_diff(&detached),
608+
model_diff(&original),
602609
),
603610
(
604611
page(&original),
605-
test_case.expected_detached.map(owned),
612+
test_case.expected_detached.map(|lines| json!(lines)),
606613
None,
607-
test_case.expected_detached.map(owned),
614+
test_case.expected_detached.map(|lines| json!(lines)),
608615
),
609616
"{}: pages read the same counts, and the text leaves the document whole",
610617
test_case.description

‎crates/sqlbuild-orchestration/src/api/_helpers/resources/changes.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ async fn stream_changes(
110110
(location_id, target): (Uuid, String),
111111
) -> Result<Response, ApiError> {
112112
let receiver = feed.sender.subscribe();
113-
let updates = watch_node_states(&cache, (location_id, &target)).await?;
113+
let (updates, watch) = watch_node_states(&cache, (location_id, &target)).await?;
114114
let state = ChangeStream {
115115
store,
116116
subscriber,
@@ -119,6 +119,7 @@ async fn stream_changes(
119119
target,
120120
receiver,
121121
updates,
122+
_watch: watch,
122123
stopped: feed.stopped,
123124
keepalive: tokio::time::interval(Duration::from_secs(15)),
124125
initial: true,

‎crates/sqlbuild-orchestration/src/api/_helpers/resources/person_reads.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@ pub(crate) async fn run_nodes(
8484

8585
#[utoipa::path(get, path = "/api/control/locations/{location_id}/targets/{target}/nodes", operation_id = "listPersonCatalogNodes",
8686
params(("location_id" = Uuid, Path), ("target" = String, Path),
87-
("If-None-Match" = Option<String>, Header, description = "An ETag from an earlier response; 304 when the snapshot and state version are unchanged")),
88-
responses((status = 200, body = Vec<CatalogNodeRecord>, description = "Every node of the current snapshot; empty before the first publication",
89-
headers(("ETag" = String, description = "\"<snapshot_id>:<state_version>\"; absent before the first publication"))),
87+
("If-None-Match" = Option<String>, Header, description = "An ETag from an earlier response; 304 when the snapshot is unchanged")),
88+
responses((status = 200, body = Vec<CatalogNodeRecord>, description = "Every node of the current snapshot without its state, which node-states serves; empty before the first publication",
89+
headers(("ETag" = String, description = "\"<snapshot_id>\"; absent before the first publication"))),
9090
(status = 304, description = "Unchanged since the If-None-Match ETag", headers(("ETag" = String))),
9191
(status = 401, description = "Session rejected"),
9292
(status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))]

0 commit comments

Comments
 (0)