Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Customers no clone touches, so --changed keeps skipping them.", materialized table);

SELECT 101 AS customer_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Orders a clone replaces in the destination target.", materialized table);

SELECT 1 AS order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Orders read downstream of the cloned table.", materialized table);

SELECT order_id FROM __ref("fact_orders")
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,16 @@
use std::collections::HashMap;

use crate::planner_identity::constants::MISSING_MESSAGE_ITEM_LIMIT;
use crate::planner_identity::models::{GraphKey, MissingUpstream};
use crate::planner_identity::models::{GraphKey, MissingUpstream, MissingUpstreamFixes};

/// Missing lineage upstreams by name and, when only an audit or hook read requires one, that read's origin.
/// Missing lineage upstreams with how to fix them and, when only an audit or hook read requires one, that read's origin.
pub fn missing_upstream_message(
missing: &[MissingUpstream],
edge_origins: &HashMap<(GraphKey, GraphKey), String>,
fixes: &MissingUpstreamFixes,
) -> String {
let mut lineage_missing: Vec<&MissingUpstream> = Vec::new();
let mut lineage_missing: Vec<&str> = Vec::new();
let mut kinds: std::collections::BTreeSet<&str> = std::collections::BTreeSet::new();
let mut read_names: Vec<&str> = Vec::new();
for entry in missing {
let origins: Vec<&str> = entry
Expand All @@ -23,7 +25,8 @@ pub fn missing_upstream_message(
})
.collect();
if origins.len() < entry.required_by.len() {
lineage_missing.push(entry);
lineage_missing.push(entry.key.name.as_str());
kinds.insert(entry.key.kind.as_str());
continue;
}
for origin in origins {
Expand All @@ -32,39 +35,121 @@ pub fn missing_upstream_message(
}
}
}
let mut parts: Vec<String> = Vec::new();
if !lineage_missing.is_empty() {
let names: Vec<&str> = lineage_missing
let reads: Option<String> = audit_reads_text(&read_names);
if lineage_missing.is_empty() {
return format!("cannot build selected scope: {}", reads.unwrap_or_default());
}
let mut text: String = lineage_text(&lineage_missing, &kinds, fixes);
if let Some(reads) = reads {
text.push_str(&format!("\n\n Also, {reads}"));
}
text
}

/// The missing lineage upstreams, a few by name, then how to read, clone or build them.
fn lineage_text(
names: &[&str],
kinds: &std::collections::BTreeSet<&str>,
fixes: &MissingUpstreamFixes,
) -> String {
let one: bool = names.len() == 1;
let noun: String = match kinds.iter().collect::<Vec<_>>().as_slice() {
[kind] if one => (*kind).to_string(),
[kind] => format!("{kind}s"),
_ => "models and seeds".to_owned(),
};
let place: String = fixes.target.as_ref().map_or_else(
|| "the warehouse".to_owned(),
|target| format!("target '{target}'"),
);
let mut lines: Vec<String> = vec![format!(
"{} upstream {noun} this build reads {} exist in {place}",
names.len(),
if one { "doesn't" } else { "don't" }
)];
lines.extend(
names
.iter()
.take(MISSING_MESSAGE_ITEM_LIMIT)
.map(|entry| entry.key.name.as_str())
.collect();
parts.push(format!(
"{} missing upstream dependencies ({})",
lineage_missing.len(),
names.join(", ")
.map(|name| format!(" {name}")),
);
if names.len() > MISSING_MESSAGE_ITEM_LIMIT {
lines.push(format!(
" ... {} more (sqb plan --json lists all)",
names.len() - MISSING_MESSAGE_ITEM_LIMIT
));
}
if let [name] = read_names.as_slice() {
parts.push(format!(
"{name}, which is not selected and does not exist in the warehouse; select it or build it first"
));
} else if !read_names.is_empty() {
let shown: String = read_names
.iter()
.take(MISSING_MESSAGE_ITEM_LIMIT)
.copied()
.collect::<Vec<&str>>()
.join("; ");
let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT {
"; ..."
lines.push(String::new());
let (exists, them): (&str, &str) = if one {
("it exists", "it")
} else {
("they exist", "them")
};
let read_label: String = format!("read {them} from it for this run:");
let copy_label: String = format!("copy {them} into this target:");
let build_label: String = format!(
"{} {them} too:",
if fixes.origins.is_empty() {
"Build"
} else {
""
};
parts.push(format!(
"{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first",
read_names.len()
"Or build"
}
);
let width: usize = if fixes.origins.is_empty() {
build_label.len() + 2
} else {
(read_label.len() + 4).max(copy_label.len() + 4)
}
.max(build_label.len() + 2);
let to: String = fixes
.clone_to
.as_ref()
.map(|target| format!(" --to {target}"))
.unwrap_or_default();
for origin in &fixes.origins {
lines.push(format!(" If {exists} in {origin}:"));
if fixes.deferrable.contains(origin) {
lines.push(format!(
" {read_label:<0$} --defer-to {origin}",
width - 4
));
}
lines.push(format!(
" {copy_label:<0$} sqb clone --from {origin}{to} -s {1}",
width - 4,
names.join(" ")
));
}
format!("cannot build selected scope: {}", parts.join("; "))
lines.push(format!(
" {build_label:<0$} add + before your selection",
width - 2
));
lines.join("\n")
}

/// The audit and hook reads that need a missing upstream, a few by name.
fn audit_reads_text(read_names: &[&str]) -> Option<String> {
match read_names {
[] => None,
[name] => Some(format!(
"{name}, which is not selected and does not exist in the warehouse; select it or build it first"
)),
_ => {
let shown: String = read_names
.iter()
.take(MISSING_MESSAGE_ITEM_LIMIT)
.copied()
.collect::<Vec<&str>>()
.join("; ");
let more: &str = if read_names.len() > MISSING_MESSAGE_ITEM_LIMIT {
"; ..."
} else {
""
};
Some(format!(
"{} audit reads are not selected and do not exist in the warehouse ({shown}{more}); select them or build them first",
read_names.len()
))
}
}
}
13 changes: 13 additions & 0 deletions crates/sqlbuild-analysis/src/planner_identity/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,19 @@ pub struct AvailableUpstreams {
pub deferred_relations: Option<HashSet<String>>,
}

/// What a missing-upstream message offers to fix the misses with, all from configuration.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct MissingUpstreamFixes {
/// The active target, named as where the upstreams are missing.
pub target: Option<String>,
/// Targets the active target may clone from, to defer to or clone the upstreams from.
pub origins: Vec<String>,
/// The origins `--defer-to` accepts: project targets, offered only when this plan does not defer already.
pub deferrable: Vec<String>,
/// The clone command's `--to`, needed when the active target is not the default one.
pub clone_to: Option<String>,
}

/// One upstream dependency missing from both scope and warehouse, with its dependents.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MissingUpstream {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use crate::planner_identity::main::missing_upstreams::missing_upstreams;
use crate::planner_identity::main::stale_upstream_warning_message::stale_upstream_warning_message;
use crate::planner_identity::main::topological_order::topological_order;
use crate::planner_identity::models::{
AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, StaleWarning,
StalenessNames,
AvailableUpstreams, FunctionFingerprintFields, GraphKey, MissingUpstream, MissingUpstreamFixes,
StaleWarning, StalenessNames,
};
use crate::planner_identity::tests::helpers::{
chain_head_hash, chain_identity_graph, chain_staleness_graph, cyclic_identity_graph,
Expand Down Expand Up @@ -406,11 +406,58 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
..AvailableUpstreams::default()
},
origins: HashMap::new(),
fixes: MissingUpstreamFixes::default(),
expected_missing: vec![MissingUpstream {
key: model("raw"),
required_by: vec![model("a"), model("b")],
}],
expected_message: "cannot build selected scope: 1 missing upstream dependencies (raw)",
expected_message: concat!(
"1 upstream model this build reads doesn't exist in the warehouse\n",
" raw\n",
"\n",
" Build it too: add + before your selection",
),
},
BuildabilityTestCase {
description: "more misses than shown, with clone origins and a non-default target",
selected: vec![model("report")],
upstream: HashMap::from([(
model("report"),
["f", "a", "e", "b", "d", "c"]
.map(model)
.into_iter()
.chain([key("seed", "codes")])
.collect(),
)]),
available: AvailableUpstreams::default(),
origins: HashMap::new(),
fixes: MissingUpstreamFixes {
target: Some("dev".to_owned()),
origins: vec!["prod".to_owned(), "staging".to_owned()],
deferrable: vec!["prod".to_owned()],
clone_to: Some("dev".to_owned()),
},
expected_missing: ["a", "b", "c", "d", "e", "f"]
.map(model)
.into_iter()
.chain([key("seed", "codes")])
.map(|key| MissingUpstream {
key,
required_by: vec![model("report")],
})
.collect(),
expected_message: concat!(
"7 upstream models and seeds this build reads don't exist in target 'dev'\n",
" a\n b\n c\n d\n e\n",
" ... 2 more (sqb plan --json lists all)\n",
"\n",
" If they exist in prod:\n",
" read them from it for this run: --defer-to prod\n",
" copy them into this target: sqb clone --from prod --to dev -s a b c d e f codes\n",
" If they exist in staging:\n",
" copy them into this target: sqb clone --from staging --to dev -s a b c d e f codes\n",
" Or build them too: add + before your selection",
),
},
BuildabilityTestCase {
description: "only an audit read requires the upstream",
Expand All @@ -421,6 +468,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
(model("a"), model("lookup")),
"audit 'check' on 'a' reads 'lookup'".to_owned(),
)]),
fixes: MissingUpstreamFixes::default(),
expected_missing: vec![MissingUpstream {
key: model("lookup"),
required_by: vec![model("a")],
Expand All @@ -436,7 +484,7 @@ fn given_selected_keys_when_checking_buildability_then_reports_missing_upstreams
);
assert_eq!(
(
missing_upstream_message(&missing, &test_case.origins),
missing_upstream_message(&missing, &test_case.origins, &test_case.fixes),
missing
),
(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet};

use crate::planner_identity::models::{
AvailableUpstreams, FunctionFingerprintFields, GraphKey, IdentityGraph, MissingUpstream,
StaleWarning, StalenessGraph,
MissingUpstreamFixes, StaleWarning, StalenessGraph,
};
use crate::planner_identity::types::IdentityHashes;

Expand Down Expand Up @@ -31,6 +31,7 @@ pub(super) struct BuildabilityTestCase {
pub(super) upstream: HashMap<GraphKey, Vec<GraphKey>>,
pub(super) available: AvailableUpstreams,
pub(super) origins: HashMap<(GraphKey, GraphKey), String>,
pub(super) fixes: MissingUpstreamFixes,
pub(super) expected_missing: Vec<MissingUpstream>,
pub(super) expected_message: &'static str,
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[
doc: "Raised when planner inputs cannot be resolved safely.",
class_attributes: &[("code", ClassValue::Text("S000"))],
required: &[],
optional: &[],
optional: &["missing"],
fixed: &[],
},
ErrorSpec {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,17 @@ pub(crate) fn new_error(
pub(crate) fn exception_from(
class: PyResult<Bound<'_, PyAny>>,
message: &str,
keywords: (Option<&str>, Option<&str>),
) -> PyErr {
exception_with_fields(class, message, keywords, &[])
}

/// The exception `class` with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it.
pub(crate) fn exception_with_fields<'py>(
class: PyResult<Bound<'py, PyAny>>,
message: &str,
(code, help): (Option<&str>, Option<&str>),
fields: &[(&str, Bound<'py, PyAny>)],
) -> PyErr {
class
.and_then(|class| {
Expand All @@ -71,6 +81,9 @@ pub(crate) fn exception_from(
if let Some(help) = help {
keywords.set_item("help", help)?;
}
for (name, value) in fields {
keywords.set_item(*name, value)?;
}
class.call((message,), Some(&keywords))
})
.map_or_else(|error| error, PyErr::from_value)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub mod error_class;
pub mod exception_from;
pub mod new_annotated_error;
pub mod new_error;
pub mod new_error_with_fields;
pub mod new_plain_error;
pub mod new_record;
pub mod new_record_with;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
//! The native `id` exception raised with `message`, its `code` and `help`, and declared keyword fields.

use crate::records::_helpers::native_classes::construct;
use crate::records::types::ErrorId;
use pyo3::prelude::{Bound, PyAny, PyErr, Python};

/// The native `id` exception with `message`, the given `code` and `help`, and its declared `fields`, or the error raising it.
#[inline]
pub fn new_error_with_fields<'py>(
py: Python<'py>,
id: ErrorId,
(message, code, help): (&str, Option<&str>, Option<&str>),
fields: &[(&str, Bound<'py, PyAny>)],
) -> PyErr {
construct::exception_with_fields(
construct::error_class(py, id),
message,
(code, help),
fields,
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ pub(crate) fn ensure_table(
}

/// Whether the warehouse catalog lists the consumption table in `location`.
fn table_exists(
pub(crate) fn table_exists(
adapter: &Bound<'_, PyAny>,
connection: &Bound<'_, PyAny>,
location: &Location,
Expand Down
Loading
Loading