Skip to content

Commit 77a12cc

Browse files
authored
fix(plan): read deferred upstreams from the deferred target and trust synced records from their last scan (#1584)
* fix(plan): read deferred upstreams from the deferred target and trust synced records from their last scan When deferring, unselected model and seed upstreams must exist in the deferred target, matched under their deferred relation names in every database the deferred locations use. S301 names the target each missing upstream was looked for in. A synced plan treats a mirrored record as current from the start of the last catalog scan that found it unchanged, timed by the server's clock, keeping the clock-skew allowance. * fix(plan): bound a synced record's scan start by when the store received it A synced plan trusted a mirrored record from the start time in its scan id, which the location's clock sets. A location clock running ahead could push that past a later change and skip describing a table that changed. Each catalog scan now stamps the records it re-delivers unchanged with the store's receipt time, and the plan trusts a record from the earlier of its scan's start and that receipt, keeping the clock-skew allowance.
1 parent 2e454f0 commit 77a12cc

31 files changed

Lines changed: 802 additions & 134 deletions

File tree

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
MODEL (description "A report a deferring build selects alone, reading the aliased orders.", materialized table);
2+
3+
SELECT order_id FROM __ref("orders")
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
MODEL (description "Orders stored under an alias, so the deferred target lists them as orders_v2.", materialized table, alias "orders_v2");
2+
3+
SELECT 1 AS order_id
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
//! The lines naming upstreams of a missing-upstream message that one target lacks.
2+
3+
use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT;
4+
5+
/// How many upstreams of these kinds are missing from `target`, a few by name.
6+
pub(crate) fn missing_lines(missing: &[(&str, &str)], target: Option<&String>) -> Vec<String> {
7+
let one: bool = missing.len() == 1;
8+
let kinds: std::collections::BTreeSet<&str> = missing.iter().map(|(_, kind)| *kind).collect();
9+
let noun: String = match kinds.iter().collect::<Vec<_>>().as_slice() {
10+
[kind] if one => (*kind).to_string(),
11+
[kind] => format!("{kind}s"),
12+
_ => "models and seeds".to_owned(),
13+
};
14+
let place: String = target.map_or_else(
15+
|| "the warehouse".to_owned(),
16+
|target| format!("target '{target}'"),
17+
);
18+
let mut lines: Vec<String> = vec![format!(
19+
"{} upstream {noun} this build reads {} exist in {place}",
20+
missing.len(),
21+
if one { "doesn't" } else { "don't" }
22+
)];
23+
lines.extend(
24+
missing
25+
.iter()
26+
.take(MISSING_MESSAGE_ITEM_LIMIT)
27+
.map(|(name, _)| format!(" {name}")),
28+
);
29+
if missing.len() > MISSING_MESSAGE_ITEM_LIMIT {
30+
lines.push(format!(
31+
" ... {} more (sqb plan --json lists all)",
32+
missing.len() - MISSING_MESSAGE_ITEM_LIMIT
33+
));
34+
}
35+
lines
36+
}

‎crates/sqlbuild-analysis/src/planner_identity/_helpers/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ pub(crate) mod compose_graph_identity;
22
pub(crate) mod cycle_message;
33
pub(crate) mod dependency_counts;
44
pub(crate) mod identity_resolver;
5+
pub(crate) mod missing_lines;
56
pub(crate) mod staleness_walk;

‎crates/sqlbuild-analysis/src/planner_identity/main/missing_upstream_message.rs‎

Lines changed: 28 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
33
use std::collections::HashMap;
44

5-
use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT;
5+
use crate::planner_identity::_helpers::missing_lines::missing_lines;
6+
7+
use crate::planner_identity::constants::{
8+
MISSING_MESSAGE_ITEM_LIMIT, MODEL_RESOURCE, SEED_RESOURCE,
9+
};
610
use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes};
711

812
/// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin.
@@ -11,8 +15,7 @@ pub fn missing_upstream_message(
1115
edge_origins: &HashMap<(GraphKey, GraphKey), String>,
1216
fixes: &MissingUpstreamFixes,
1317
) -> String {
14-
let mut lineage_missing: Vec<&str> = Vec::new();
15-
let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
18+
let mut lineage_missing: Vec<(&str, &str)> = Vec::new();
1619
let mut read_names: Vec<&str> = Vec::new();
1720
for entry in missing {
1821
let origins: Vec<&str> = entry
@@ -25,8 +28,7 @@ pub fn missing_upstream_message(
2528
})
2629
.collect();
2730
if origins.len() < entry.required_by.len() {
28-
lineage_missing.push(entry.key.name.as_str());
29-
kinds.insert(entry.key.kind.as_str());
31+
lineage_missing.push((entry.key.name.as_str(), entry.key.kind.as_str()));
3032
continue;
3133
}
3234
for origin in origins {
@@ -39,46 +41,35 @@ pub fn missing_upstream_message(
3941
if lineage_missing.is_empty() {
4042
return format!("cannot build selected scope: {}", reads.unwrap_or_default());
4143
}
42-
let mut text: String = lineage_text(&lineage_missing, &kinds, fixes);
44+
let mut text: String = lineage_text(&lineage_missing, fixes);
4345
if let Some(reads) = reads {
4446
text.push_str(&format!("\n\n Also, {reads}"));
4547
}
4648
text
4749
}
4850

49-
/// The missing lineage upstreams, a few by name, then how to read, clone or build them.
50-
fn lineage_text(
51-
names: &[&str],
52-
kinds: &std::collections::BTreeSet<&str>,
53-
fixes: &MissingUpstreamFixes,
54-
) -> String {
51+
/// Missing lineage upstreams per target looked in (deferred for models and seeds when deferring), then the fixes.
52+
fn lineage_text(missing: &[(&str, &str)], fixes: &MissingUpstreamFixes) -> String {
53+
let names: Vec<&str> = missing.iter().map(|(name, _)| *name).collect();
5554
let one: bool = names.len() == 1;
56-
let noun: String = match kinds.iter().collect::<Vec<_>>().as_slice() {
57-
[kind] if one => (*kind).to_string(),
58-
[kind] => format!("{kind}s"),
59-
_ => "models and seeds".to_owned(),
55+
let groups: Vec<Vec<String>> = match fixes.deferred_to.as_ref() {
56+
Some(deferred_to) => {
57+
let (deferred, active): (Vec<_>, Vec<_>) = missing
58+
.iter()
59+
.copied()
60+
.partition(|(_, kind)| [MODEL_RESOURCE, SEED_RESOURCE].contains(kind));
61+
[
62+
(deferred, Some(deferred_to)),
63+
(active, fixes.target.as_ref()),
64+
]
65+
.into_iter()
66+
.filter(|(group, _)| !group.is_empty())
67+
.map(|(group, target)| missing_lines(&group, target))
68+
.collect()
69+
}
70+
None => vec![missing_lines(missing, fixes.target.as_ref())],
6071
};
61-
let place: String = fixes.target.as_ref().map_or_else(
62-
|| "the warehouse".to_owned(),
63-
|target| format!("target '{target}'"),
64-
);
65-
let mut lines: Vec<String> = vec![format!(
66-
"{} upstream {noun} this build reads {} exist in {place}",
67-
names.len(),
68-
if one { "doesn't" } else { "don't" }
69-
)];
70-
lines.extend(
71-
names
72-
.iter()
73-
.take(MISSING_MESSAGE_ITEM_LIMIT)
74-
.map(|name| format!(" {name}")),
75-
);
76-
if names.len() > MISSING_MESSAGE_ITEM_LIMIT {
77-
lines.push(format!(
78-
" ... {} more (sqb plan --json lists all)",
79-
names.len() - MISSING_MESSAGE_ITEM_LIMIT
80-
));
81-
}
72+
let mut lines: Vec<String> = groups.join(&String::new());
8273
lines.push(String::new());
8374
let (exists, them): (&str, &str) = if one {
8475
("it exists", "it")

‎crates/sqlbuild-analysis/src/planner_identity/main/missing_upstreams.rs‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
33
use std::collections::{HashMap, HashSet};
44

5-
use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE};
5+
use crate::planner_identity::constants::{
6+
DBT_REF_RESOURCE, MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE,
7+
};
68
use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream};
79

8-
/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`.
10+
/// Missing upstreams, most dependents first; a deferring plan checks unselected models and seeds in the deferred target.
911
pub fn missing_upstreams(
1012
selected_keys: &[GraphKey],
1113
upstream_deps: &HashMap<GraphKey, Vec<GraphKey>>,
@@ -19,11 +21,14 @@ pub fn missing_upstreams(
1921
.contains(&dep_key.kind.as_str())
2022
|| selected.contains(dep_key)
2123
|| available.satisfied_keys.contains(dep_key)
22-
|| available.existing_relations.contains(&dep_key.name)
23-
|| available
24-
.deferred_relations
25-
.as_ref()
26-
.is_some_and(|deferred| deferred.contains(&dep_key.name))
24+
|| match &available.deferred_relations {
25+
Some(deferred)
26+
if [MODEL_RESOURCE, SEED_RESOURCE].contains(&dep_key.kind.as_str()) =>
27+
{
28+
deferred.contains(&dep_key.name)
29+
}
30+
_ => available.existing_relations.contains(&dep_key.name),
31+
}
2732
{
2833
continue;
2934
}

‎crates/sqlbuild-analysis/src/planner_identity/models.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ pub struct AvailableUpstreams {
9292
pub struct MissingUpstreamFixes {
9393
/// The active target, named as where the upstreams are missing.
9494
pub target: Option<String>,
95+
/// The target a deferring plan reads unselected models and seeds from, named as where those are missing.
96+
pub deferred_to: Option<String>,
9597
/// Targets the active target may clone from, to defer to or clone the upstreams from.
9698
pub origins: Vec<String>,
9799
/// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already.

‎crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
436436
origins: vec!["prod".to_owned(), "staging".to_owned()],
437437
deferrable: vec!["prod".to_owned()],
438438
clone_to: Some("dev".to_owned()),
439+
deferred_to: None,
439440
},
440441
expected_missing: ["a", "b", "c", "d", "e", "f"]
441442
.map(model)
@@ -459,6 +460,73 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
459460
" Or build them too: add + before your selection",
460461
),
461462
},
463+
BuildabilityTestCase {
464+
description: "deferring reads models and seeds only from the deferred target",
465+
selected: vec![model("report")],
466+
upstream: HashMap::from([(
467+
model("report"),
468+
vec![model("dev_only"), model("prod_only"), key("seed", "codes")],
469+
)]),
470+
available: AvailableUpstreams {
471+
existing_relations: names(&["dev_only", "codes"]),
472+
deferred_relations: Some(names(&["prod_only", "codes"])),
473+
..AvailableUpstreams::default()
474+
},
475+
origins: HashMap::new(),
476+
fixes: MissingUpstreamFixes {
477+
target: Some("dev".to_owned()),
478+
deferred_to: Some("prod".to_owned()),
479+
..MissingUpstreamFixes::default()
480+
},
481+
expected_missing: vec![MissingUpstream {
482+
key: model("dev_only"),
483+
required_by: vec![model("report")],
484+
}],
485+
expected_message: concat!(
486+
"1 upstream model this build reads doesn't exist in target 'prod'\n",
487+
" dev_only\n",
488+
"\n",
489+
" Build it too: add + before your selection",
490+
),
491+
},
492+
BuildabilityTestCase {
493+
description: "deferring names the deferred target for models and the active one for functions",
494+
selected: vec![model("report")],
495+
upstream: HashMap::from([(
496+
model("report"),
497+
vec![model("dev_only"), key("udf", "parse_sku")],
498+
)]),
499+
available: AvailableUpstreams {
500+
existing_relations: names(&["dev_only"]),
501+
deferred_relations: Some(names(&["parse_sku"])),
502+
..AvailableUpstreams::default()
503+
},
504+
origins: HashMap::new(),
505+
fixes: MissingUpstreamFixes {
506+
target: Some("dev".to_owned()),
507+
deferred_to: Some("prod".to_owned()),
508+
..MissingUpstreamFixes::default()
509+
},
510+
expected_missing: vec![
511+
MissingUpstream {
512+
key: model("dev_only"),
513+
required_by: vec![model("report")],
514+
},
515+
MissingUpstream {
516+
key: key("udf", "parse_sku"),
517+
required_by: vec![model("report")],
518+
},
519+
],
520+
expected_message: concat!(
521+
"1 upstream model this build reads doesn't exist in target 'prod'\n",
522+
" dev_only\n",
523+
"\n",
524+
"1 upstream udf this build reads doesn't exist in target 'dev'\n",
525+
" parse_sku\n",
526+
"\n",
527+
" Build them too: add + before your selection",
528+
),
529+
},
462530
BuildabilityTestCase {
463531
description: "only an audit read requires the upstream",
464532
selected: vec![model("a")],

‎crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -745,7 +745,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync
745745
state.progress.scan_unchanged = false;
746746
state.progress.scan_started = Instant::now();
747747
state.index += 1;
748-
state.scan_id = Uuid::now_v7();
748+
state.scan_id = state.server_scan_id();
749749
state.observed_since = None;
750750
state.watermark_loaded = false;
751751
state.limit = MAX_HISTORY_PAGE_RECORDS;

‎crates/sqlbuild-cli/src/native_commands/models.rs‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{
1414
MetadataBatch, StatisticsRead, WarehouseHistoryFailure, WarehouseHistoryKind,
1515
WarehouseHistoryPage, WarehouseStateLocation,
1616
};
17-
use uuid::Uuid;
17+
use uuid::{NoContext, Timestamp, Uuid};
1818

1919
use crate::argv::models::Namespace;
2020
use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_POLL_SECONDS};
@@ -425,12 +425,24 @@ impl LocationMetadataSync {
425425
self.reads_wake_compute && !self.safety_explicit
426426
}
427427

428+
/// A scan id timed by the server's clock: a synced plan trusts a record as of its scan's start, never after a read.
429+
pub(crate) fn server_scan_id(&self) -> Uuid {
430+
let now: DateTime<Utc> = Utc::now() + self.server_offset;
431+
let seconds: u64 = u64::try_from(now.timestamp()).unwrap_or_default();
432+
Uuid::new_v7(Timestamp::from_unix(
433+
NoContext,
434+
seconds,
435+
now.timestamp_subsec_nanos(),
436+
))
437+
}
438+
428439
/// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass.
429440
pub(crate) fn begin_sync(&mut self, now: DateTime<Utc>) {
441+
self.server_offset = now - Utc::now();
442+
self.scan_id = self.server_scan_id();
430443
if std::mem::take(&mut self.halted) {
431444
self.index = 0;
432445
self.source_after = None;
433-
self.scan_id = Uuid::now_v7();
434446
self.observed_since = None;
435447
self.watermark_loaded = false;
436448
self.statistics = None;
@@ -441,7 +453,6 @@ impl LocationMetadataSync {
441453
self.applied = false;
442454
}
443455
self.idle = false;
444-
self.server_offset = now - Utc::now();
445456
self.progress = SyncProgress::new();
446457
self.cycle_started_at = Some(now);
447458
self.next_at = tokio::time::Instant::now();
@@ -470,7 +481,7 @@ impl LocationMetadataSync {
470481
if statistics {
471482
self.index -= 1;
472483
self.source_after = None;
473-
self.scan_id = Uuid::now_v7();
484+
self.scan_id = self.server_scan_id();
474485
self.observed_since = None;
475486
self.watermark_loaded = false;
476487
}

0 commit comments

Comments
 (0)