diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py index 80d336fbbb..ce5a9b6723 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -13,3 +13,13 @@ ) def fulfillment_calendar(window, fulfillment): return fulfillment.upcoming(window) + + +# Declared for another target without decorating a function: development publication skips it. +production_calendar = calendar_schedule( + job=orders, + description="Production fulfillment times, not yet attached to a provider.", + horizon="8s", + timezone="UTC", + targets=("production",), +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 31884a29df..fd2e5a038d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1854,7 +1854,7 @@ export interface components { }; JobMember: { job_key: string; - /** @description Members this one waits for, from lineage and from the group's `after`. */ + /** @description Members this one waits for, from the lineage between their nodes. */ waits_for: string[]; }; /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index 76a2b097aa..fcc0b6ece8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: an explicit `after`, or the first lineage edge. +/// Keep the first reason found for a wait: the first lineage edge. pub(crate) fn record_wait( mut reasons: BTreeMap<(String, String), String>, (job, dependency): (&str, &str), diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index d2103f2439..6e78d95cfb 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -1,4 +1,4 @@ -//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. +//! Freeze job boundaries from snapshot lineage, rejecting cycles. use std::collections::{BTreeMap, BTreeSet}; @@ -10,19 +10,12 @@ use crate::store::errors::ControlError; pub fn group_dependencies( (group, input): (&str, &SnapshotInput), jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> Result>, ControlError> { - if members.is_empty() - || members.iter().any(|name| !jobs.contains_key(name)) - || after - .iter() - .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) - { - return Err(ControlError::Invalid( - "group needs known jobs and dependencies within its membership".into(), - )); + if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) { + return Err(ControlError::Invalid("group needs known jobs".into())); } - let waits = lineage_dependencies(input, jobs, (members, after)); + let waits = lineage_dependencies(input, jobs, members); ensure_acyclic(group, &waits.edges, &waits.reasons)?; Ok(waits.edges) } @@ -31,7 +24,7 @@ pub fn group_dependencies( fn lineage_dependencies( input: &SnapshotInput, jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> GroupWaits { let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for name in members { @@ -42,12 +35,7 @@ fn lineage_dependencies( let mut edges: BTreeMap> = BTreeMap::new(); let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); for name in members { - let mut dependencies = after.get(name).cloned().unwrap_or_default(); - for dependency in &dependencies { - reasons = record_wait(reasons, (name, dependency), || { - "declared in after".to_owned() - }); - } + let mut dependencies: BTreeSet = BTreeSet::new(); let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index a415f62d13..4d42f2baa4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -30,8 +30,7 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); - let dependencies = - group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?; + let dependencies = group_dependencies((name, input), &definitions.jobs, &members)?; let selection: BTreeSet = group .jobs .keys() diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 96428acafa..8af0a32820 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -256,7 +256,7 @@ pub struct JobDetailRecord { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct JobMember { pub job_key: String, - /// Members this one waits for, from lineage and from the group's `after`. + /// Members this one waits for, from the lineage between their nodes. pub waits_for: BTreeSet, } @@ -343,7 +343,7 @@ pub(crate) struct PreviousTriggerState { pub(crate) status_override: Option, } -/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found. +/// Each member job's waits, and why it waits: the first lineage edge found. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct GroupWaits { pub(crate) edges: BTreeMap>, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 6f90bc7572..643b151e5c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -491,12 +491,17 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for (index, parent) in test_case.parents.iter().enumerate() { let mut members = serde_json::Map::new(); let mut selection = BTreeSet::new(); + let mut earlier_nodes: Vec<&str> = Vec::new(); for (stage, needs) in parent.children.iter().enumerate() { let name = format!("orders_job_{index}_{stage}"); let node = ["orders", "customers", "inventory", "products"][child_count % 4]; child_count += 1; selection.insert(node); + // Each stage reads every earlier stage's node, so lineage orders the stages. + input.nodes.get_mut(node).expect("stage node").definition["deps"] = + json!(earlier_nodes); + earlier_nodes.push(node); jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index cef10c013d..82d54108a1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -152,6 +152,11 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a input.generation = 2; input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = json!(["inventory"]); + input + .nodes + .get_mut("inventory") + .expect("inventory") + .definition["deps"] = json!(["customers"]); input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = json!(["customers", "inventory"]); publish_snapshot(store, &input) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index e8586b2850..f0cbec653a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -87,8 +87,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r .nodes .get_mut("customers") .expect("customers node") - .definition = - json!({"materialization": "table", "config": {"cursor_type": "timestamp"}}); + .definition = json!({"materialization": "table", "deps": ["orders"], "config": {"cursor_type": "timestamp"}}); publish_snapshot(store, &input).await.expect("publish jobs"); let administrator = (&roles.context, credentials(&roles.administrator)); let location = roles.location_id; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs index 34c162e5b1..26cae64df5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs @@ -102,6 +102,12 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n ip_address: None, }; let mut input = snapshot(location_id); + // Customers read orders, so customer_sync waits for daily_orders. + input + .nodes + .get_mut("customers") + .expect("customers model") + .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"], "concurrency_groups": ["orders-load"]}, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 7fff917906..339285d199 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -68,13 +68,23 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .get_mut("orders") .expect("orders model") .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); + // Lineage orders the group: shipments reads orders and inventory reads products, so + // marts waits for staging only through completed, never directly. + let mut shipments = input.nodes["products"].clone(); + shipments.definition["deps"] = json!(["orders"]); + input.nodes.insert("shipments".into(), shipments); + input + .nodes + .get_mut("inventory") + .expect("inventory model") + .definition["deps"] = json!(["products"]); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, - "completed": {"description": "Build products", "selection": ["products"]} + "completed": {"description": "Build products and shipments", "selection": ["products", "shipments"]} })); input.documents.insert("groups".into(), json!({"daily_orders": { - "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products", "shipments"], "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); @@ -131,6 +141,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("admit source"); let node_outcomes = BTreeMap::from([ ("products", NodeOutcome::Success), + ("shipments", NodeOutcome::Success), ("orders", NodeOutcome::Failed), ("customers", NodeOutcome::Cancelled), ("inventory", NodeOutcome::Skipped), @@ -265,9 +276,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..retry.clone() }, expected_applied: true, - expected_selection: ["orders", "inventory", "customers", "products"] - .map(str::to_owned) - .into(), + expected_selection: [ + "orders", + "inventory", + "customers", + "products", + "shipments", + ] + .map(str::to_owned) + .into(), expected_jobs: &["staging", "completed", "marts"], }, ] { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 3e5a9123ba..34e0873df8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -668,7 +668,9 @@ pub(crate) fn run_post_build_python_checks<'py>( ) -> PyResult> { let py: Python<'py> = context.py(); let result = &outcome.result; - if result.status != BuildStatus::Success { + if result.status != BuildStatus::Success + || !native_field(&context.request, "run_audits")?.extract::()? + { return tuple(py, &[]); } let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs index 8f6eef3210..4339117ee9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs @@ -226,6 +226,8 @@ pub(crate) struct DiagnosticSettings { pub(crate) loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. pub(crate) python_resources: Vec, + /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. + pub(crate) description_limit: (usize, i64), } /// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 194c416314..47304d5c3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -25,8 +25,11 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ module_jobs, run_controls, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - compile_calendars, compile_cron, compile_sensors, + calendar_descriptions, compile_calendars, compile_cron, compile_sensors, }; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; /// Dedicated declaration modules, imported by publication and by selectors that name a job. pub(crate) fn discover_declarations<'py>( @@ -87,6 +90,39 @@ pub(crate) fn compile_declarations( AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), } } + let format: FormatSettings = + resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; + let mut described: Vec<(&str, String, String)> = Vec::new(); + described.extend( + authored_jobs + .iter() + .map(|job| ("job", job.name.clone(), job.description.clone())), + ); + described.extend( + authored_groups + .iter() + .map(|group| ("job group", group.name.clone(), group.description.clone())), + ); + described.extend( + authored_sensors + .iter() + .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), + ); + described.extend( + calendar_descriptions(py, &authored_calendars)? + .into_iter() + .map(|(name, description)| ("calendar schedule", name, description)), + ); + for (kind, name, description) in described { + let limit: i64 = format.max_description_lines; + let lines: usize = description_line_count(&description, format.line_width); + if i64::try_from(lines).unwrap_or(i64::MAX) > limit { + return Err(PyValueError::new_err(format!( + "{kind} '{name}' description exceeds {limit} lines; shorten it to {limit} lines \ + and move detail into documentation" + ))); + } + } let mut triggers: BTreeMap = BTreeMap::new(); for job in authored_jobs { if !job.targets.is_empty() && !job.targets.contains(&input.target) { @@ -167,17 +203,7 @@ pub(crate) fn compile_declarations( if members.len() != group.jobs.len() { return Err(PyValueError::new_err("group members must be distinct")); } - let mut after: BTreeMap> = BTreeMap::new(); - for (job, dependencies) in group.after { - after.insert( - job.name, - dependencies - .into_iter() - .map(|dependency| dependency.name) - .collect(), - ); - } - let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after)) + let dependencies = group_dependencies((&group.name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index d34298388a..bf3340298c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -1,13 +1,18 @@ //! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. use std::collections::HashMap; +use std::path::PathBuf; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; +use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES}; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow; @@ -564,6 +569,7 @@ fn diagnostic_settings<'py>( .collect::>()?; settings.set_item("sql_hooks", hooks)?; settings.set_item("loaders", loader_rows(discovered)?)?; + settings.set_item("description_limit", description_limit(discovered)?)?; settings.set_item( "python_resources", call_with( @@ -577,6 +583,23 @@ fn diagnostic_settings<'py>( Ok(settings) } +/// The project's `max_description_lines` and `line_width` as `sqb format` resolves them, else the defaults. +fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { + let directory: Option = native_field(discovered, "project_dir")?.extract()?; + let Some(directory) = directory else { + return Ok(( + usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0), + DEFAULT_LINE_WIDTH, + )); + }; + let settings: FormatSettings = + resolve_format_settings(&directory).map_err(PyValueError::new_err)?; + Ok(( + usize::try_from(settings.max_description_lines).unwrap_or(0), + settings.line_width, + )) +} + /// Attach every SQL test case, scenario and audit, then report every resource diagnostic. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index cae1f4f929..577882795c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -19,6 +19,7 @@ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; use crate::bindings::_helpers::compile_context::delta::{ @@ -35,6 +36,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_project::models::ReferenceRow; const MISSING_DESCRIPTION_CODE: &str = "P010"; +const DESCRIPTION_LENGTH_CODE: &str = "P027"; const CURSOR_WITHOUT_INPUTS_CODE: &str = "P011"; const ORPHAN_FACTORY_CODE: &str = "C216"; const AUDIT_FACTORIES_KEY: &str = "audit_factories"; @@ -134,21 +136,35 @@ impl<'py> Diagnostics<'_, 'py> { .call((), Some(&fields)) } - /// P010 for `kind` `name` when its description is missing or blank. + /// P010 when `kind` `name` has no description; for Python, P027 when it wraps past the limit. fn described( &self, resource: (&str, &str, Option<&str>), - description: Option<&str>, + (description, python): (Option<&str>, bool), location: impl FnOnce() -> PyResult>, help: &str, ) -> PyResult>> { let (kind, name, resource_type) = resource; - let Some(message) = missing_description(kind, name, description) else { - return Ok(None); + let (code, message, help): (&str, String, String) = if let Some(message) = + missing_description(kind, name, description) + { + (MISSING_DESCRIPTION_CODE, message, help.to_owned()) + } else { + let (limit, line_width) = self.settings.description_limit; + match description { + Some(text) if python && description_line_count(text, line_width) > limit => ( + DESCRIPTION_LENGTH_CODE, + format!("{kind} '{name}' description exceeds {limit} lines"), + format!( + "Shorten the description to {limit} lines; move detail into documentation." + ), + ), + _ => return Ok(None), + } }; Ok(Some(self.diagnostic( - (MISSING_DESCRIPTION_CODE, "error"), - (&message, Some(help)), + (code, "error"), + (&message, Some(&help)), (resource_type, name), location()?, )?)) @@ -271,7 +287,7 @@ fn missing_descriptions<'py>( .and_then(|schema| schema.description.as_deref()); found.push(diagnostics.described( ("model", &name, Some("model")), - description, + (description, false), || diagnostics.header_location(&path, contents), &help, )?); @@ -300,7 +316,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("scenario", name, Some("sql_scenario")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -322,7 +338,7 @@ fn missing_descriptions<'py>( let contents = contents_value.downcast::()?.to_str()?; found.push(diagnostics.described( ("seed", &name, Some("seed")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, &name), &yaml_help("seed", &name, &posix(&path)), )?); @@ -351,7 +367,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("hook", name, None), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -369,7 +385,7 @@ fn missing_descriptions<'py>( .map(|description| docstring_text(description.bind(py))); found.push(diagnostics.described( (kind, name, None), - text.as_deref(), + (text.as_deref(), true), || location.bind(py).call0(), help, )?); @@ -423,7 +439,7 @@ fn source_description<'py>( } diagnostics.described( ("source", name, Some("source")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, name), &help, ) @@ -458,7 +474,7 @@ fn function_description<'py>( ); return diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), true), || { let version = py.version_info(); let position: (usize, usize) = decorator_location( @@ -482,7 +498,7 @@ fn function_description<'py>( }); diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), false), || diagnostics.header_location(&path, &contents), &help, ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index dcf1dfd9bc..ae92c19465 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -50,7 +50,6 @@ pub(crate) struct AuthoredGroup { pub(crate) name: String, pub(crate) description: String, pub(crate) jobs: Vec, - pub(crate) after: BTreeMap>, pub(crate) concurrency_groups: Vec, pub(crate) schedule: Option>, pub(crate) targets: Vec, @@ -359,20 +358,6 @@ pub(crate) fn collect_declarations<'py>( } pending.push(member); } - for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { - if !member.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - for dependency in dependencies.try_iter()? { - if !dependency?.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - } - } AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index b8923950a0..9faf31f8e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -184,6 +184,25 @@ fn calendar_times( }) } +/// The name and description of each calendar that decorates a function. +pub(crate) fn calendar_descriptions( + py: Python<'_>, + calendars: &[Py], +) -> PyResult> { + let mut described: Vec<(String, String)> = Vec::new(); + for calendar in calendars { + let calendar: &Bound<'_, PyAny> = calendar.bind(py); + let function: Bound<'_, PyAny> = calendar.getattr("function")?; + if !function.is_none() { + described.push(( + function.getattr("__name__")?.extract()?, + calendar.getattr("description")?.extract()?, + )); + } + } + Ok(described) +} + pub(crate) fn compile_calendars( py: Python<'_>, (project_dir, target): (&Path, &str), diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs new file mode 100644 index 0000000000..fb3ca72df0 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs @@ -0,0 +1,10 @@ +//! `description_line_count`: the lines a description takes once wrapped at the line width. + +use crate::dsl_headers::_helpers::text::line_count; +use crate::dsl_headers::_helpers::wrap::wrap_description; + +/// The lines `description` takes wrapped at `line_width`, as the description-length rule counts them. +pub fn description_line_count(description: &str, line_width: i64) -> usize { + let characters: Vec = description.chars().collect(); + line_count(&wrap_description(&characters, line_width, line_width)) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs index 862b8dc0f6..19633251d9 100644 --- a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs +++ b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs @@ -1,4 +1,5 @@ pub mod declares_table_returns; +pub mod description_line_count; pub mod fix_escaped_quotes; pub mod format_headers; pub mod inner_header_text; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 3ead104853..a629cf9be8 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -169,8 +169,8 @@ pub(crate) const NON_CONTIGUOUS_FIX: &str = pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; -pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; +pub const DEFAULT_LINE_WIDTH: i64 = 100; +pub const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; pub(crate) const DEFAULT_DIALECT: &str = "generic"; diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 67cf616bf2..c0922abdc4 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -169,6 +169,19 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="python-node-description-too-long", + expected_code="P027", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\ndef export_orders(ctx: TaskContext):\n" + ' """' + + "\n\n ".join(f"Orders paragraph {index}." for index in range(6)) + + '"""\n return None\n' + ) + }, + ), failure_case( name="enum-parse", expected_code="D013", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 4626809a05..351c64faf4 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -574,6 +574,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "P023", "P024", "P025", + "P027", "S026", } ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 01cf27cd3e..1c09acc5b3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -53,13 +53,19 @@ formatted. `--fix` never skips files. Pass ## Line width and descriptions -Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width and the most lines a description may take in +`sqlbuild_project.toml` (the defaults are `100` and `10`): ```toml [format] line_width = 100 +max_description_lines = 10 ``` +A header description longer than `max_description_lines`, once wrapped at the line width, fails +`sqb format --check` with `description-length`. Python descriptions keep to the same limit at +compile; see [Descriptions](../concepts/models/configuration.md#descriptions). + `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers need a non-empty `description`. When one is missing, `sqb format --check` fails with `description-present`, so pre-commit catches it before compile does. A model described by its `model_schema` or its `[path_defaults]` entry passes, as it diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 5213641a9f..4add747a77 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -87,6 +87,11 @@ error[P010]: model 'orders_summary' has no description | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | +A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped +at the line width, the same limit `sqb format --check` applies to headers. A longer Python +description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, +job group, sensor or calendar schedule with a longer description cannot publish. + SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 3c98c3c3c4..ddb45ff27b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -764,6 +764,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | +| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](selectors.md#job) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index f127ece7e5..0bf9122d9f 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -63,12 +63,11 @@ def __post_init__(self) -> None: @dataclass(frozen=True, eq=False) class JobGroup: - """Jobs ordered by compiled lineage and optional explicit dependencies.""" + """Jobs ordered by the lineage between the nodes they build.""" name: str description: str = field(kw_only=True) jobs: Sequence[Job] = field(kw_only=True) - after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 712fe48b83..8140e645d2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -318,6 +318,18 @@ def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) """ ).strip() + "\n", + "python/checks/orders.py": dedent( + """ + from sqlbuild.checks import check + from sqlbuild.refs import model + + + @check(depends_on=model("orders")) + def check_orders_exist(ctx): + '''Test check check_orders_exist.''' + return ctx.pass_() + """ + ).lstrip(), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3af739ac24..d7fd693423 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -257,14 +257,7 @@ def export_check(ctx): db_path: Path = project_dir / "factory_nodes_project.duckdb" run_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--no-tests", - "--no-audits", - "--select", - "+asset:orders_export", - ), + command=("--no-color", "build", "--no-tests", "--select", "+asset:orders_export"), project_dir=project_dir, ) bare_result: subprocess.CompletedProcess[str] = run_sqb( @@ -2260,14 +2253,7 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no ) result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--no-tests", - "--no-audits", - "--select", - *test_case.select, - ), + command=("--no-color", "build", "--no-tests", "--select", *test_case.select), project_dir=project_dir, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py index 467368d178..ee86817ce3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py @@ -21,15 +21,15 @@ description="no tests skips SQL tests but runs audits", project_name="build_no_tests_project", command=("--no-color", "build", "--no-tests"), - expected_stdout_fragments=("audit not_null",), + expected_stdout_fragments=("audit not_null", "check check_orders_exist"), unexpected_stdout_fragments=("test test_orders",), ), BuildNoTestsNoAuditsFlagE2ETestCase( - description="no audits skips audits but runs SQL tests", + description="no audits skips audits and Python checks but runs SQL tests", project_name="build_no_audits_project", command=("--no-color", "build", "--no-audits"), expected_stdout_fragments=("test test_orders",), - unexpected_stdout_fragments=("audit not_null",), + unexpected_stdout_fragments=("audit not_null", "check_orders_exist"), ), ), ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py index fd91edd38b..3d560ca5d7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py @@ -178,7 +178,7 @@ project_kind="terminal_loader", ), CheckCommandTestCase( - description="build without tests or audits still executes Python checks", + description="build without tests or audits skips Python checks", command=( "--no-color", "build", @@ -188,11 +188,8 @@ "tag:failure", ), expected_returncode=0, - expected_stdout_fragments=( - "Execution sqb build", - "Python checks", - "check_orders_export", - ), + expected_stdout_fragments=("Execution sqb build",), + expected_absent_fragments=("Python checks", "check_orders_export"), ), CheckCommandTestCase( description="audit remains SQL audit only", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index e7e6b97035..d3cfbc3b53 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3265,6 +3265,13 @@ def hooked_model_file(*, model_name: str, hook: str) -> tuple[str, str]: ) +def description_paragraphs(count: int) -> str: + """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" + + body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) + return f' """{body}"""\n' + + def python_node_source( *, module: str, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index eab3e5a839..fedac2da3e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -22,6 +22,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( REQUIRED_DESCRIPTIONS_PROJECT, compile_inline_files, + description_paragraphs, hooked_model_file, python_node_source, require_sql_analysis_output, @@ -506,6 +507,88 @@ ) +# The default max_description_lines is 10: five paragraphs fit in 9 lines, six need 11. +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionCase( + description="task docstring", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(6), + ), + ), + ), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(5), + ), + ), + ), + expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), + RequiredDescriptionCase( + description="asset description argument", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 90!r})", + ), + ), + ), + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 60!r})", + ), + ), + ), + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_description_past_the_line_limit_when_compiling_then_reports_p027( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + too_long: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "too_long", files=test_case.missing_files + ) + at_limit: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "at_limit", files=test_case.described_files + ) + + diagnostics, text = require_sql_analysis_output(too_long) + assert too_long.returncode == 1, too_long.stdout + too_long.stderr + assert diagnostics == (test_case.expected_diagnostic,) + assert json.loads(too_long.stdout)["diagnostics"][0]["message"] == test_case.expected_message + assert all(fragment in text for fragment in test_case.expected_help_fragments), text + assert at_limit.returncode == 0, at_limit.stdout + at_limit.stderr + assert require_sql_analysis_output(at_limit)[0] == () + + @pytest.mark.parametrize( "test_case", [RequiredDescriptionCase(**asdict(case)) for case in _CASES], diff --git a/tests/goldens/compiler/failure/python-node-description-too-long.json b/tests/goldens/compiler/failure/python-node-description-too-long.json new file mode 100644 index 0000000000..34bd8ae483 --- /dev/null +++ b/tests/goldens/compiler/failure/python-node-description-too-long.json @@ -0,0 +1,271 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P027", + "message": "task 'export_orders' description exceeds 10 lines", + "resource_name": "export_orders", + "path": "python/tasks/exports.py", + "line": 4, + "column": 1, + "location": { + "path": "python/tasks/exports.py", + "line": 4, + "column": 1 + }, + "help": "Shorten the description to 10 lines; move detail into documentation." + } + ] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 179999d185..06e61dc873 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1102,7 +1102,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject expected_jobs=("daily", "staging"), ), SnapshotDiscoveryTestCase( - description="group-only references retain lineage and explicit waits", + description="group-only references retain lineage waits", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" @@ -1111,14 +1111,14 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject " marts = job('marts', description='Build orders.', select='orders')\n" " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" " return job_group('daily', description='Ordered jobs.', " - "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "jobs=[marts, inventory, staging])\n" "daily = factory()\n" ), }, expected_jobs=("inventory", "marts", "staging"), expected_groups={ "daily": { - "inventory": ["staging"], + "inventory": [], "marts": ["inventory", "staging"], "staging": [], } @@ -1505,35 +1505,34 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_message="callbacks with when are not supported yet", ), SnapshotDiscoveryRejectionTestCase( - description="explicit group cycles cannot publish", + description="a job description past max_description_lines cannot publish", files={ "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" - "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Cyclic jobs.', " - "jobs=[marts, staging], after={staging: [marts]})\n" + "from sqlbuild.orchestration import job\n" + "daily = job('daily', description='Build orders. ' * 100, select='orders')\n" ), }, expected_error=ValueError, expected_message=re.escape( - "group dependencies contain a cycle in job group daily: marts waits for staging " - "(model:orders reads model:stg_orders); staging waits for marts (declared in after)" + "job 'daily' description exceeds 10 lines; shorten it to 10 lines" ), ), SnapshotDiscoveryRejectionTestCase( - description="group waits cannot refer to nonmember jobs", + description="a group declaring after is refused at its module", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" "staging = job('staging', description='Stage orders.', select='stg_orders')\n" "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Unknown upstream.', " - "jobs=[marts], after={marts: [staging]})\n" + "daily = job_group('daily', description='Ordered jobs.', " + "jobs=[marts, staging], after={staging: [marts]})\n" ), }, expected_error=ValueError, - expected_message="group needs known jobs and dependencies within its membership", + expected_message=re.escape( + "Failed to import orchestration file orchestration/jobs.py: JobGroup.__init__() " + "got an unexpected keyword argument 'after'" + ), ), SnapshotDiscoveryRejectionTestCase( description="distinct objects cannot share a job name", diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index ab0a7b05a2..aacde0be9f 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1155,7 +1155,6 @@ export class MockOperate { if (a === b) continue; if ([...nodesOf.get(a)!].some((n) => needs.has(n))) ups.push({ jobKey: a, source: 'lineage' }); } - for (const a of job.after[b] ?? []) if (!ups.some((u) => u.jobKey === a)) ups.push({ jobKey: a, source: 'after' }); upstream.set(b, ups); } const stage = new Map(); @@ -1176,10 +1175,8 @@ export class MockOperate { const view = latest ? this.#view(latest, scope) : null; const holders = this.#records.filter((r) => HOLDING.has(r.run.status)); const resolved = job.kind === 'group' ? jobNodes(key) : resolveJobSelection(job).nodes; - const { after, ...rest } = job; - void after; return structuredClone({ - ...rest, + ...job, nodeCount: resolved.filter((n) => snapshotNode(n)?.kind !== 'source').length, resolvedNodes: resolved, lastRun: view ? { id: view.id, status: view.status, startedAt: view.startedAt, endedAt: view.endedAt } : null, diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 28c0da7c9f..168130a8fb 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -37,9 +37,7 @@ type Known = Omit & { [P in K]-?: NonNullable type JobDef = Known< Omit, 'members' | 'overlap' | 'priority' | 'sourceFile' | 'declaration' | 'settings' -> & { - after: Record; -}; +>; export const CONCURRENCY_LIMITS: Record = { warehouse_heavy: 1, 'target:prod': 4 }; @@ -62,7 +60,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'ingest', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 14, @@ -91,7 +88,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/jobs.py', 24, @@ -124,7 +120,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod', 'staging'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 37, @@ -153,7 +148,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/activity.py', 8, @@ -184,7 +178,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 11, @@ -215,7 +208,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 23, @@ -243,7 +235,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/reporting.py', 6, @@ -273,7 +264,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/groups.py', 5, @@ -303,7 +293,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: ['daily_revenue'], - after: { customer_segments: ['revenue_refresh'] }, declaration: decl( 'orchestration/groups.py', 16, @@ -311,8 +300,6 @@ export const JOB_DEFS: JobDef[] = [ "close_of_business", description="Revenue first, then activity and customer segments", jobs=[revenue_refresh, activity_microbatch, customer_segments], - # Segments read closed revenue through an external lookup lineage can't see. - after={customer_segments: [revenue_refresh]}, on_failure=on_failure(retries=1, retry_delay_seconds=900), critical=["daily_revenue"], concurrency_groups=[warehouse_heavy], diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c8f98f9ec2..0dc7087afa 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -688,8 +688,8 @@ export type ConcurrencyGroup = { export type JobGroupMember = { jobKey: string; - /** Jobs this member waits for: lineage-derived plus declared `after`; null where the cause isn't reported. */ - upstream: { jobKey: string; source: 'lineage' | 'after' | null }[]; + /** Jobs this member waits for, from lineage; null where the cause isn't reported. */ + upstream: { jobKey: string; source: 'lineage' | null }[]; /** Position in the derived order; members in the same stage run side by side. */ stage: number; }; diff --git a/ui/src/lib/lineage/job-graph.test.ts b/ui/src/lib/lineage/job-graph.test.ts index eac4067156..00f4749e9c 100644 --- a/ui/src/lib/lineage/job-graph.test.ts +++ b/ui/src/lib/lineage/job-graph.test.ts @@ -6,7 +6,7 @@ import { boxId, jobGraph, layoutJobGraph } from './job-graph'; const node = (key: string, upstream: string[] = []) => ({ key, kind: 'table', upstream, tags: [], stale: null, checks: null }) as unknown as CatalogNode; const measured = (nodes: Node[]) => nodes.map((n) => (n.type === 'job' ? n : { ...n, measured: { width: 236, height: 61 } })); -// staging builds raw → stg; orders reads stg; reports waits for orders only through `after`. +// staging builds raw → stg; orders reads stg; reports waits for orders through lineage outside the graph. const catalog = [node('model:raw'), node('model:stg', ['model:raw']), node('model:orders', ['model:stg']), node('model:report')]; const all = catalog.map((n) => n.key); const boxes = [ @@ -17,11 +17,11 @@ const boxes = [ const members = [ { jobKey: 'staging', upstream: [], stage: 0 }, { jobKey: 'orders', upstream: [{ jobKey: 'staging', source: 'lineage' as const }], stage: 1 }, - { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: 'after' as const }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } + { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: null }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } ]; describe('job graph', () => { - it('puts each node in the first box that builds it and draws `after` only where no lineage links two boxes', () => { + it('puts each node in the first box that builds it and draws a wait edge only where no drawn lineage links two boxes', () => { const { nodes, edges } = jobGraph(all, catalog, boxes, members); const parent = Object.fromEntries(nodes.filter((n) => n.type === 'asset').map((n) => [n.id, n.parentId])); expect(parent).toEqual({ 'model:raw': boxId('staging'), 'model:stg': boxId('staging'), 'model:orders': boxId('orders'), 'model:report': boxId('reports') }); @@ -64,7 +64,7 @@ describe('job graph', () => { expect(nodes.filter((n) => n.type === 'asset').map((n) => n.id).sort()).toEqual(['model:orders', 'model:report']); expect(nodes.find((n) => n.id === boxId('staging'))!.data.collapsed).toBe(true); // raw → stg sits inside the box and disappears; stg → orders now leaves the box. Staging and - // reports still have no lineage between them, so their `after` edge stays. + // reports still have no lineage between them, so their wait edge stays. expect(edges.map((e) => `${e.source}>${e.target}`).sort()).toEqual( [`${boxId('staging')}>model:orders`, `${boxId('orders')}>${boxId('reports')}`, `${boxId('staging')}>${boxId('reports')}`].sort() ); @@ -106,8 +106,8 @@ describe('job graph', () => { { jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) }, { jobKey: 'empty', nodes: [] } ]; - const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 }); - const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]); + const waiting = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: null }], stage: 1 }); + const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, waiting('b'), waiting('empty')]); const laid = layoutJobGraph(measured(nodes), edges); const box = (k: string) => laid.find((n) => n.id === boxId(k))!; expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60); diff --git a/ui/src/lib/lineage/job-graph.ts b/ui/src/lib/lineage/job-graph.ts index 4917d59c7d..b4d9ae3d8c 100644 --- a/ui/src/lib/lineage/job-graph.ts +++ b/ui/src/lib/lineage/job-graph.ts @@ -26,7 +26,7 @@ const MARGIN = 24; export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null }; const edgeStyle = 'stroke: var(--sb-edge); stroke-width: 1.5;'; -const afterStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; +const waitStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; export function boxId(jobKey: string): string { return `job:${jobKey}`; @@ -35,8 +35,9 @@ export function boxId(jobKey: string): string { /** * Cards for `nodes` and the lineage between them. With `boxes`, a node built by several members sits * in the most specific box, the one building the fewest nodes (the earliest on a tie), so a small - * job inside a broad one keeps its nodes; a declared `after` between two members with no lineage - * between them is a dashed edge between their boxes. A box in `collapsed` shows only its header, + * job inside a broad one keeps its nodes; a member that waits for another with no drawn lineage + * between them, because the lineage runs through nodes outside the graph, gets a dashed edge between + * their boxes. A box in `collapsed` shows only its header, * and lineage to its cards meets the box instead. */ export function jobGraph( @@ -99,13 +100,13 @@ export function jobGraph( ); const jobOf = (id: string) => owner.get(id) ?? id.slice('job:'.length); const linked = new Set(lineage.map((e) => `${jobOf(e.source)}>${jobOf(e.target)}`)); - const after: Edge[] = members.flatMap((m) => + const waits: Edge[] = members.flatMap((m) => m.upstream .filter((u) => !linked.has(`${u.jobKey}>${m.jobKey}`) && boxes.some((b) => b.jobKey === u.jobKey)) - .map((u) => ({ id: `after:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: afterStyle })) + .map((u) => ({ id: `wait:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: waitStyle })) ); // Svelte Flow needs a parent before its children. - return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] }; + return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...waits] }; } /** The most cards a column holds before it wraps, growing with the graph so its shape stays near the canvas's. */ @@ -200,7 +201,7 @@ function layoutLayer(items: Item[], pairs: ReadonlyArray