Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "A report a deferring build selects alone, reading the aliased orders.", materialized table);

SELECT order_id FROM __ref("orders")
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Orders stored under an alias, so the deferred target lists them as orders_v2.", materialized table, alias "orders_v2");

SELECT 1 AS order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
//! The lines naming upstreams of a missing-upstream message that one target lacks.

use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT;

/// How many upstreams of these kinds are missing from `target`, a few by name.
pub(crate) fn missing_lines(missing: &[(&str, &str)], target: Option<&String>) -> Vec<String> {
let one: bool = missing.len() == 1;
let kinds: std::collections::BTreeSet<&str> = missing.iter().map(|(_, kind)| *kind).collect();
let noun: String = match kinds.iter().collect::<Vec<_>>().as_slice() {
[kind] if one => (*kind).to_string(),
[kind] => format!("{kind}s"),
_ => "models and seeds".to_owned(),
};
let place: String = target.map_or_else(
|| "the warehouse".to_owned(),
|target| format!("target '{target}'"),
);
let mut lines: Vec<String> = vec![format!(
"{} upstream {noun} this build reads {} exist in {place}",
missing.len(),
if one { "doesn't" } else { "don't" }
)];
lines.extend(
missing
.iter()
.take(MISSING_MESSAGE_ITEM_LIMIT)
.map(|(name, _)| format!(" {name}")),
);
if missing.len() > MISSING_MESSAGE_ITEM_LIMIT {
lines.push(format!(
" ... {} more (sqb plan --json lists all)",
missing.len() - MISSING_MESSAGE_ITEM_LIMIT
));
}
lines
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@ pub(crate) mod compose_graph_identity;
pub(crate) mod cycle_message;
pub(crate) mod dependency_counts;
pub(crate) mod identity_resolver;
pub(crate) mod missing_lines;
pub(crate) mod staleness_walk;
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@

use std::collections::HashMap;

use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT;
use crate::planner_identity::_helpers::missing_lines::missing_lines;

use crate::planner_identity::constants::{
MISSING_MESSAGE_ITEM_LIMIT, MODEL_RESOURCE, SEED_RESOURCE,
};
use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes};

/// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin.
Expand All @@ -11,8 +15,7 @@ pub fn missing_upstream_message(
edge_origins: &HashMap<(GraphKey, GraphKey), String>,
fixes: &MissingUpstreamFixes,
) -> String {
let mut lineage_missing: Vec<&str> = Vec::new();
let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
let mut lineage_missing: Vec<(&str, &str)> = Vec::new();
let mut read_names: Vec<&str> = Vec::new();
for entry in missing {
let origins: Vec<&str> = entry
Expand All @@ -25,8 +28,7 @@ pub fn missing_upstream_message(
})
.collect();
if origins.len() < entry.required_by.len() {
lineage_missing.push(entry.key.name.as_str());
kinds.insert(entry.key.kind.as_str());
lineage_missing.push((entry.key.name.as_str(), entry.key.kind.as_str()));
continue;
}
for origin in origins {
Expand All @@ -39,46 +41,35 @@ pub fn missing_upstream_message(
if lineage_missing.is_empty() {
return format!("cannot build selected scope: {}", reads.unwrap_or_default());
}
let mut text: String = lineage_text(&lineage_missing, &kinds, fixes);
let mut text: String = lineage_text(&lineage_missing, fixes);
if let Some(reads) = reads {
text.push_str(&format!("\n\n Also, {reads}"));
}
text
}

/// The missing lineage upstreams, a few by name, then how to read, clone or build them.
fn lineage_text(
names: &[&str],
kinds: &std::collections::BTreeSet<&str>,
fixes: &MissingUpstreamFixes,
) -> String {
/// Missing lineage upstreams per target looked in (deferred for models and seeds when deferring), then the fixes.
fn lineage_text(missing: &[(&str, &str)], fixes: &MissingUpstreamFixes) -> String {
let names: Vec<&str> = missing.iter().map(|(name, _)| *name).collect();
let one: bool = names.len() == 1;
let noun: String = match kinds.iter().collect::<Vec<_>>().as_slice() {
[kind] if one => (*kind).to_string(),
[kind] => format!("{kind}s"),
_ => "models and seeds".to_owned(),
let groups: Vec<Vec<String>> = match fixes.deferred_to.as_ref() {
Some(deferred_to) => {
let (deferred, active): (Vec<_>, Vec<_>) = missing
.iter()
.copied()
.partition(|(_, kind)| [MODEL_RESOURCE, SEED_RESOURCE].contains(kind));
[
(deferred, Some(deferred_to)),
(active, fixes.target.as_ref()),
]
.into_iter()
.filter(|(group, _)| !group.is_empty())
.map(|(group, target)| missing_lines(&group, target))
.collect()
}
None => vec![missing_lines(missing, fixes.target.as_ref())],
};
let place: String = fixes.target.as_ref().map_or_else(
|| "the warehouse".to_owned(),
|target| format!("target '{target}'"),
);
let mut lines: Vec<String> = vec![format!(
"{} upstream {noun} this build reads {} exist in {place}",
names.len(),
if one { "doesn't" } else { "don't" }
)];
lines.extend(
names
.iter()
.take(MISSING_MESSAGE_ITEM_LIMIT)
.map(|name| format!(" {name}")),
);
if names.len() > MISSING_MESSAGE_ITEM_LIMIT {
lines.push(format!(
" ... {} more (sqb plan --json lists all)",
names.len() - MISSING_MESSAGE_ITEM_LIMIT
));
}
let mut lines: Vec<String> = groups.join(&String::new());
lines.push(String::new());
let (exists, them): (&str, &str) = if one {
("it exists", "it")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@

use std::collections::{HashMap, HashSet};

use crate::planner_identity::constants::{DBT_REF_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE};
use crate::planner_identity::constants::{
DBT_REF_RESOURCE, MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE, SQL_TEST_RESOURCE,
};
use crate::planner_identity::models::{AvailableUpstreams, GraphKey, MissingUpstream};

/// Missing upstreams, most dependents first, then by `(kind, name)`; dependents sorted by `(kind, name)`.
/// Missing upstreams, most dependents first; a deferring plan checks unselected models and seeds in the deferred target.
pub fn missing_upstreams(
selected_keys: &[GraphKey],
upstream_deps: &HashMap<GraphKey, Vec<GraphKey>>,
Expand All @@ -19,11 +21,14 @@ pub fn missing_upstreams(
.contains(&dep_key.kind.as_str())
|| selected.contains(dep_key)
|| available.satisfied_keys.contains(dep_key)
|| available.existing_relations.contains(&dep_key.name)
|| available
.deferred_relations
.as_ref()
.is_some_and(|deferred| deferred.contains(&dep_key.name))
|| match &available.deferred_relations {
Some(deferred)
if [MODEL_RESOURCE, SEED_RESOURCE].contains(&dep_key.kind.as_str()) =>
{
deferred.contains(&dep_key.name)
}
_ => available.existing_relations.contains(&dep_key.name),
}
{
continue;
}
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-analysis/src/planner_identity/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ pub struct AvailableUpstreams {
pub struct MissingUpstreamFixes {
/// The active target, named as where the upstreams are missing.
pub target: Option<String>,
/// The target a deferring plan reads unselected models and seeds from, named as where those are missing.
pub deferred_to: Option<String>,
/// Targets the active target may clone from, to defer to or clone the upstreams from.
pub origins: Vec<String>,
/// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
origins: vec!["prod".to_owned(), "staging".to_owned()],
deferrable: vec!["prod".to_owned()],
clone_to: Some("dev".to_owned()),
deferred_to: None,
},
expected_missing: ["a", "b", "c", "d", "e", "f"]
.map(model)
Expand All @@ -459,6 +460,73 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
" Or build them too: add + before your selection",
),
},
BuildabilityTestCase {
description: "deferring reads models and seeds only from the deferred target",
selected: vec![model("report")],
upstream: HashMap::from([(
model("report"),
vec![model("dev_only"), model("prod_only"), key("seed", "codes")],
)]),
available: AvailableUpstreams {
existing_relations: names(&["dev_only", "codes"]),
deferred_relations: Some(names(&["prod_only", "codes"])),
..AvailableUpstreams::default()
},
origins: HashMap::new(),
fixes: MissingUpstreamFixes {
target: Some("dev".to_owned()),
deferred_to: Some("prod".to_owned()),
..MissingUpstreamFixes::default()
},
expected_missing: vec![MissingUpstream {
key: model("dev_only"),
required_by: vec![model("report")],
}],
expected_message: concat!(
"1 upstream model this build reads doesn't exist in target 'prod'\n",
" dev_only\n",
"\n",
" Build it too: add + before your selection",
),
},
BuildabilityTestCase {
description: "deferring names the deferred target for models and the active one for functions",
selected: vec![model("report")],
upstream: HashMap::from([(
model("report"),
vec![model("dev_only"), key("udf", "parse_sku")],
)]),
available: AvailableUpstreams {
existing_relations: names(&["dev_only"]),
deferred_relations: Some(names(&["parse_sku"])),
..AvailableUpstreams::default()
},
origins: HashMap::new(),
fixes: MissingUpstreamFixes {
target: Some("dev".to_owned()),
deferred_to: Some("prod".to_owned()),
..MissingUpstreamFixes::default()
},
expected_missing: vec![
MissingUpstream {
key: model("dev_only"),
required_by: vec![model("report")],
},
MissingUpstream {
key: key("udf", "parse_sku"),
required_by: vec![model("report")],
},
],
expected_message: concat!(
"1 upstream model this build reads doesn't exist in target 'prod'\n",
" dev_only\n",
"\n",
"1 upstream udf this build reads doesn't exist in target 'dev'\n",
" parse_sku\n",
"\n",
" Build them too: add + before your selection",
),
},
BuildabilityTestCase {
description: "only an audit read requires the upstream",
selected: vec![model("a")],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -745,7 +745,7 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync
state.progress.scan_unchanged = false;
state.progress.scan_started = Instant::now();
state.index += 1;
state.scan_id = Uuid::now_v7();
state.scan_id = state.server_scan_id();
state.observed_since = None;
state.watermark_loaded = false;
state.limit = MAX_HISTORY_PAGE_RECORDS;
Expand Down
19 changes: 15 additions & 4 deletions crates/sqlbuild-cli/src/native_commands/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use sqlbuild_orchestration::warehouse_metadata::models::{
MetadataBatch, StatisticsRead, WarehouseHistoryFailure, WarehouseHistoryKind,
WarehouseHistoryPage, WarehouseStateLocation,
};
use uuid::Uuid;
use uuid::{NoContext, Timestamp, Uuid};

use crate::argv::models::Namespace;
use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_POLL_SECONDS};
Expand Down Expand Up @@ -425,12 +425,24 @@ impl LocationMetadataSync {
self.reads_wake_compute && !self.safety_explicit
}

/// A scan id timed by the server's clock: a synced plan trusts a record as of its scan's start, never after a read.
pub(crate) fn server_scan_id(&self) -> Uuid {
let now: DateTime<Utc> = Utc::now() + self.server_offset;
let seconds: u64 = u64::try_from(now.timestamp()).unwrap_or_default();
Uuid::new_v7(Timestamp::from_unix(
NoContext,
seconds,
now.timestamp_subsec_nanos(),
))
}

/// Start the sync `sync_due` asked for at server time `now`; a stopped sync restarts its pass.
pub(crate) fn begin_sync(&mut self, now: DateTime<Utc>) {
self.server_offset = now - Utc::now();
self.scan_id = self.server_scan_id();
if std::mem::take(&mut self.halted) {
self.index = 0;
self.source_after = None;
self.scan_id = Uuid::now_v7();
self.observed_since = None;
self.watermark_loaded = false;
self.statistics = None;
Expand All @@ -441,7 +453,6 @@ impl LocationMetadataSync {
self.applied = false;
}
self.idle = false;
self.server_offset = now - Utc::now();
self.progress = SyncProgress::new();
self.cycle_started_at = Some(now);
self.next_at = tokio::time::Instant::now();
Expand Down Expand Up @@ -470,7 +481,7 @@ impl LocationMetadataSync {
if statistics {
self.index -= 1;
self.source_after = None;
self.scan_id = Uuid::now_v7();
self.scan_id = self.server_scan_id();
self.observed_since = None;
self.watermark_loaded = false;
}
Expand Down
Loading
Loading