Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
aadbc98
test: reproduce Python selectors in published job selections
kvlonge Nov 14, 2026
889f3d4
wip: resolve published Python selectors with the CLI's unified resolver
kvlonge Nov 14, 2026
cc4a6c5
feat: let a selected Python node read an unselected upstream's result
kvlonge Nov 14, 2026
0e665a9
feat: select Python-only jobs and SQL tests across CLI and server runs
kvlonge Nov 14, 2026
d78eb7e
Merge remote-tracking branch 'origin/main' into feat/python-nodes-in-…
kvlonge Nov 14, 2026
a96db64
fix: keep Python selections runnable on workers and require checked n…
kvlonge Nov 14, 2026
879f931
fix: refuse selecting a Python check on a source no loader writes
kvlonge Nov 14, 2026
059f8cd
refactor: share upstream result reads between loader and node contexts
kvlonge Nov 14, 2026
e0c01ba
feat!: order job groups by lineage only and skip checks under --no-au…
kvlonge Nov 14, 2026
62a2241
docs: list orchestration/ in the Project Python table
kvlonge Nov 14, 2026
c6d7661
feat: apply max_description_lines to Python and orchestration descrip…
kvlonge Nov 14, 2026
b5d8823
Merge remote-tracking branch 'origin/main' into feat/python-nodes-in-…
kvlonge Nov 14, 2026
20956f1
Merge branch 'feat/python-nodes-in-snapshot' into feat/orchestration-…
kvlonge Nov 14, 2026
1e3e9e8
fix: keep orchestration fixtures and description limits consistent wi…
kvlonge Nov 14, 2026
66f2d7d
test: keep the rerun wait-boundary case transitive under lineage-only…
kvlonge Nov 14, 2026
c949c54
refactor: report P010 and P027 from one description check
kvlonge Nov 14, 2026
cc210da
chore: regenerate the control client for lineage-only group waits
kvlonge Nov 14, 2026
eed4c55
refactor: route calendar descriptions through the native host and dro…
kvlonge Nov 14, 2026
3b40dcc
test: expect the unknown-argument error for JobGroup after
kvlonge Nov 14, 2026
0cdaba6
test: cover P027 in the compiler failure corpus
kvlonge Nov 14, 2026
84365df
test: record the golden for the P027 failure corpus case
kvlonge Nov 14, 2026
ebe1b7a
test: expect build --no-audits to skip Python checks in the E2E suites
kvlonge Nov 14, 2026
c481831
Merge remote-tracking branch 'origin/main' into HEAD
kvlonge Nov 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,13 @@
)
def fulfillment_calendar(window, fulfillment):
return fulfillment.upcoming(window)


# Declared for another target without decorating a function: development publication skips it.
production_calendar = calendar_schedule(
job=orders,
description="Production fulfillment times, not yet attached to a provider.",
horizon="8s",
timezone="UTC",
targets=("production",),
)
2 changes: 1 addition & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1854,7 +1854,7 @@ export interface components {
};
JobMember: {
job_key: string;
/** @description Members this one waits for, from lineage and from the group's `after`. */
/** @description Members this one waits for, from the lineage between their nodes. */
waits_for: string[];
};
/** @description A job or group of the current published snapshot. Settings are left out of person reads. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use std::collections::{BTreeMap, BTreeSet};

/// Keep the first reason found for a wait: an explicit `after`, or the first lineage edge.
/// Keep the first reason found for a wait: the first lineage edge.
pub(crate) fn record_wait(
mut reasons: BTreeMap<(String, String), String>,
(job, dependency): (&str, &str),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles.
//! Freeze job boundaries from snapshot lineage, rejecting cycles.

use std::collections::{BTreeMap, BTreeSet};

Expand All @@ -10,19 +10,12 @@ use crate::store::errors::ControlError;
pub fn group_dependencies(
(group, input): (&str, &SnapshotInput),
jobs: &BTreeMap<String, JobDefinition>,
(members, after): (&BTreeSet<String>, &BTreeMap<String, BTreeSet<String>>),
members: &BTreeSet<String>,
) -> Result<BTreeMap<String, BTreeSet<String>>, ControlError> {
if members.is_empty()
|| members.iter().any(|name| !jobs.contains_key(name))
|| after
.iter()
.any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members))
{
return Err(ControlError::Invalid(
"group needs known jobs and dependencies within its membership".into(),
));
if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) {
return Err(ControlError::Invalid("group needs known jobs".into()));
}
let waits = lineage_dependencies(input, jobs, (members, after));
let waits = lineage_dependencies(input, jobs, members);
ensure_acyclic(group, &waits.edges, &waits.reasons)?;
Ok(waits.edges)
}
Expand All @@ -31,7 +24,7 @@ pub fn group_dependencies(
fn lineage_dependencies(
input: &SnapshotInput,
jobs: &BTreeMap<String, JobDefinition>,
(members, after): (&BTreeSet<String>, &BTreeMap<String, BTreeSet<String>>),
members: &BTreeSet<String>,
) -> GroupWaits {
let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
for name in members {
Expand All @@ -42,12 +35,7 @@ fn lineage_dependencies(
let mut edges: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new();
for name in members {
let mut dependencies = after.get(name).cloned().unwrap_or_default();
for dependency in &dependencies {
reasons = record_wait(reasons, (name, dependency), || {
"declared in after".to_owned()
});
}
let mut dependencies: BTreeSet<String> = BTreeSet::new();
let mut seen: BTreeSet<&str> = BTreeSet::new();
let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect();
while let Some(key) = pending.pop() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result<Definitions, Contro
}
for (name, group) in &definitions.groups {
let members: BTreeSet<String> = group.jobs.keys().cloned().collect();
let dependencies =
group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?;
let dependencies = group_dependencies((name, input), &definitions.jobs, &members)?;
let selection: BTreeSet<String> = group
.jobs
.keys()
Expand Down
4 changes: 2 additions & 2 deletions crates/sqlbuild-orchestration/src/declarations/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ pub struct JobDetailRecord {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
pub struct JobMember {
pub job_key: String,
/// Members this one waits for, from lineage and from the group's `after`.
/// Members this one waits for, from the lineage between their nodes.
pub waits_for: BTreeSet<String>,
}

Expand Down Expand Up @@ -343,7 +343,7 @@ pub(crate) struct PreviousTriggerState {
pub(crate) status_override: Option<TriggerStatus>,
}

/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found.
/// Each member job's waits, and why it waits: the first lineage edge found.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct GroupWaits {
pub(crate) edges: BTreeMap<String, BTreeSet<String>>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -491,12 +491,17 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th
for (index, parent) in test_case.parents.iter().enumerate() {
let mut members = serde_json::Map::new();
let mut selection = BTreeSet::new();
let mut earlier_nodes: Vec<&str> = Vec::new();
for (stage, needs) in parent.children.iter().enumerate() {
let name = format!("orders_job_{index}_{stage}");
let node =
["orders", "customers", "inventory", "products"][child_count % 4];
child_count += 1;
selection.insert(node);
// Each stage reads every earlier stage's node, so lineage orders the stages.
input.nodes.get_mut(node).expect("stage node").definition["deps"] =
json!(earlier_nodes);
earlier_nodes.push(node);
jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs}));
let after: Vec<String> = (0..stage)
.map(|earlier| format!("orders_job_{index}_{earlier}"))
Expand Down
5 changes: 5 additions & 0 deletions crates/sqlbuild-orchestration/tests/runs/test_groups.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,11 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a
input.generation = 2;
input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] =
json!(["inventory"]);
input
.nodes
.get_mut("inventory")
.expect("inventory")
.definition["deps"] = json!(["customers"]);
input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] =
json!(["customers", "inventory"]);
publish_snapshot(store, &input)
Expand Down
3 changes: 1 addition & 2 deletions crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
.nodes
.get_mut("customers")
.expect("customers node")
.definition =
json!({"materialization": "table", "config": {"cursor_type": "timestamp"}});
.definition = json!({"materialization": "table", "deps": ["orders"], "config": {"cursor_type": "timestamp"}});
publish_snapshot(store, &input).await.expect("publish jobs");
let administrator = (&roles.context, credentials(&roles.administrator));
let location = roles.location_id;
Expand Down
6 changes: 6 additions & 0 deletions crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n
ip_address: None,
};
let mut input = snapshot(location_id);
// Customers read orders, so customer_sync waits for daily_orders.
input
.nodes
.get_mut("customers")
.expect("customers model")
.definition["deps"] = json!(["orders"]);
input.documents.insert("jobs".into(), json!({
"daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"],
"concurrency_groups": ["orders-load"]},
Expand Down
27 changes: 22 additions & 5 deletions crates/sqlbuild-orchestration/tests/runs/test_reruns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,23 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_
.get_mut("orders")
.expect("orders model")
.definition = json!({"materialization": "table", "deps": ["source:raw_orders"]});
// Lineage orders the group: shipments reads orders and inventory reads products, so
// marts waits for staging only through completed, never directly.
let mut shipments = input.nodes["products"].clone();
shipments.definition["deps"] = json!(["orders"]);
input.nodes.insert("shipments".into(), shipments);
input
.nodes
.get_mut("inventory")
.expect("inventory model")
.definition["deps"] = json!(["products"]);
input.documents.insert("jobs".into(), json!({
"staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}},
"marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]},
"completed": {"description": "Build products", "selection": ["products"]}
"completed": {"description": "Build products and shipments", "selection": ["products", "shipments"]}
}));
input.documents.insert("groups".into(), json!({"daily_orders": {
"description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"],
"description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products", "shipments"],
"concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]}
}}));
let published = publish_snapshot(store, &input).await.expect("publish");
Expand Down Expand Up @@ -131,6 +141,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_
.expect("admit source");
let node_outcomes = BTreeMap::from([
("products", NodeOutcome::Success),
("shipments", NodeOutcome::Success),
("orders", NodeOutcome::Failed),
("customers", NodeOutcome::Cancelled),
("inventory", NodeOutcome::Skipped),
Expand Down Expand Up @@ -265,9 +276,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_
..retry.clone()
},
expected_applied: true,
expected_selection: ["orders", "inventory", "customers", "products"]
.map(str::to_owned)
.into(),
expected_selection: [
"orders",
"inventory",
"customers",
"products",
"shipments",
]
.map(str::to_owned)
.into(),
expected_jobs: &["staging", "completed", "marts"],
},
] {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,9 @@ pub(crate) fn run_post_build_python_checks<'py>(
) -> PyResult<Bound<'py, PyAny>> {
let py: Python<'py> = context.py();
let result = &outcome.result;
if result.status != BuildStatus::Success {
if result.status != BuildStatus::Success
|| !native_field(&context.request, "run_audits")?.extract::<bool>()?
{
return tuple(py, &[]);
}
let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,8 @@ pub(crate) struct DiagnosticSettings {
pub(crate) loaders: HashMap<String, LoaderRow>,
/// Python-defined resources: kind, name, description, location and help, read by Python.
pub(crate) python_resources: Vec<PythonResourceRow>,
/// The project's `max_description_lines` and `line_width`, which Python descriptions keep to.
pub(crate) description_limit: (usize, i64),
}

/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{
module_jobs, run_controls,
};
use crate::bindings::_helpers::native_host::orchestration_triggers::{
compile_calendars, compile_cron, compile_sensors,
calendar_descriptions, compile_calendars, compile_cron, compile_sensors,
};
use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count;
use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings;
use sqlbuild_rules::lint_text::models::FormatSettings;

/// Dedicated declaration modules, imported by publication and by selectors that name a job.
pub(crate) fn discover_declarations<'py>(
Expand Down Expand Up @@ -87,6 +90,39 @@ pub(crate) fn compile_declarations(
AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor),
}
}
let format: FormatSettings =
resolve_format_settings(project_dir).map_err(PyValueError::new_err)?;
let mut described: Vec<(&str, String, String)> = Vec::new();
described.extend(
authored_jobs
.iter()
.map(|job| ("job", job.name.clone(), job.description.clone())),
);
described.extend(
authored_groups
.iter()
.map(|group| ("job group", group.name.clone(), group.description.clone())),
);
described.extend(
authored_sensors
.iter()
.map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())),
);
described.extend(
calendar_descriptions(py, &authored_calendars)?
.into_iter()
.map(|(name, description)| ("calendar schedule", name, description)),
);
for (kind, name, description) in described {
let limit: i64 = format.max_description_lines;
let lines: usize = description_line_count(&description, format.line_width);
if i64::try_from(lines).unwrap_or(i64::MAX) > limit {
return Err(PyValueError::new_err(format!(
"{kind} '{name}' description exceeds {limit} lines; shorten it to {limit} lines \
and move detail into documentation"
)));
}
}
let mut triggers: BTreeMap<String, TriggerDefinition> = BTreeMap::new();
for job in authored_jobs {
if !job.targets.is_empty() && !job.targets.contains(&input.target) {
Expand Down Expand Up @@ -167,17 +203,7 @@ pub(crate) fn compile_declarations(
if members.len() != group.jobs.len() {
return Err(PyValueError::new_err("group members must be distinct"));
}
let mut after: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for (job, dependencies) in group.after {
after.insert(
job.name,
dependencies
.into_iter()
.map(|dependency| dependency.name)
.collect(),
);
}
let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after))
let dependencies = group_dependencies((&group.name, &input), &jobs, &members)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
let selection: BTreeSet<String> = members
.iter()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
//! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs.

use std::collections::HashMap;
use std::path::PathBuf;

use pyo3::exceptions::PyValueError;
use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python};
use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple};
use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction};
use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS;
use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing;
use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost};
use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES};
use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings;
use sqlbuild_rules::lint_text::models::FormatSettings;

use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported};
use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow;
Expand Down Expand Up @@ -564,6 +569,7 @@ fn diagnostic_settings<'py>(
.collect::<PyResult<_>>()?;
settings.set_item("sql_hooks", hooks)?;
settings.set_item("loaders", loader_rows(discovered)?)?;
settings.set_item("description_limit", description_limit(discovered)?)?;
settings.set_item(
"python_resources",
call_with(
Expand All @@ -577,6 +583,23 @@ fn diagnostic_settings<'py>(
Ok(settings)
}

/// The project's `max_description_lines` and `line_width` as `sqb format` resolves them, else the defaults.
fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> {
let directory: Option<PathBuf> = native_field(discovered, "project_dir")?.extract()?;
let Some(directory) = directory else {
return Ok((
usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0),
DEFAULT_LINE_WIDTH,
));
};
let settings: FormatSettings =
resolve_format_settings(&directory).map_err(PyValueError::new_err)?;
Ok((
usize::try_from(settings.max_description_lines).unwrap_or(0),
settings.line_width,
))
}

/// Attach every SQL test case, scenario and audit, then report every resource diagnostic.
#[pyfunction]
#[pyo3(signature = (**options))]
Expand Down
Loading
Loading