From 4b5939f80fc0794d52e401dc2cbe1779ea135332 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:59:26 +0000 Subject: [PATCH 1/5] feat(server): plan ad-hoc --vars, --load, --reload and load, gated by run.vars and run.load (WIP) --- .../src/adhoc/_helpers/planner.rs | 69 ++++++++++--- crates/sqlbuild-cli/src/adhoc/constants.rs | 39 +++++--- .../src/adhoc/tests/test_adhoc_planner.rs | 69 +++++++++++-- .../authorization/_helpers/person_grants.rs | 4 +- .../src/credentials/models.rs | 4 + .../src/runs/_helpers/requests/adhoc.rs | 10 +- .../src/runs/_helpers/requests/permissions.rs | 38 ++++++- .../tests/runs/test_adhoc.rs | 98 ++++++++++++++++--- .../tests/runs/test_types.rs | 21 ++++ 9 files changed, 304 insertions(+), 48 deletions(-) diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 241a2eaf88..255154d2c4 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -20,9 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, - FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, - POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, - UNIT_TEST_VERBS, + FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_BUILD_SETTINGS, + LOAD_SOURCES_DEST, LOAD_VERB, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, RELOAD_DEST, + SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, VARS_CONVERTER, VARS_DEST, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -44,7 +44,7 @@ impl CommandPlanner for CliPlanner { .ok_or_else(|| "enter a command, such as build --select orders".to_owned())?; if !ADHOC_VERBS.contains(&verb.as_str()) { return Err(format!( - "sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit or freshness" + "sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit, freshness or load" )); } let parsed = parse(command)?; @@ -66,9 +66,22 @@ impl CommandPlanner for CliPlanner { if dest == CHANGED_DEST && settings.contains_key(CHANGED_DEST) { continue; } - settings.insert(dest.clone(), setting(dest, value)?); + let setting = setting(dest, value)?; + if dest == VARS_DEST && setting.as_object().is_some_and(serde_json::Map::is_empty) { + continue; + } + settings.insert(dest.clone(), setting); + } + if verb == LOAD_VERB { + for (dest, value) in LOAD_BUILD_SETTINGS { + settings.insert(dest.to_owned(), json!(value)); + } + // `--reload` implies loading, and a build refuses it together with `--load`. + if settings.get(RELOAD_DEST) == Some(&Json::Bool(true)) { + settings.remove(LOAD_SOURCES_DEST); + } } - if verb == BUILD_VERB { + if verb == BUILD_VERB || verb == LOAD_VERB { job_settings_arguments(&settings)?; } else { command_settings_arguments(verb, &settings)?; @@ -88,6 +101,7 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, + LOAD_VERB => managed_sources(publication, &select, &exclude)?, _ => { let python = verb == BUILD_VERB && settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false)); @@ -102,11 +116,22 @@ impl CommandPlanner for CliPlanner { } } -/// Whole numbers convert as the parser needs them; paths and variables stay text so the refusal can name them. +/// Whole numbers convert as the parser needs them; `--vars` must be a JSON object; paths stay text so the refusal can name them. struct PlannerHost; impl ArgvHost for PlannerHost { fn convert(&mut self, converter: &str, text: &str) -> Result { + if converter == VARS_CONVERTER { + return match serde_json::from_str::(text) { + Ok(Json::Object(_)) => Ok(Value::Str(text.to_owned())), + Ok(_) => Err(TypeFailure::ArgumentType( + "--vars must be a JSON object".to_owned(), + )), + Err(error) => Err(TypeFailure::ArgumentType(format!( + "--vars must be valid JSON: {error}" + ))), + }; + } if !INTEGER_CONVERTERS.contains(&converter) { return Ok(Value::Str(text.to_owned())); } @@ -165,12 +190,6 @@ fn setting(dest: &str, value: &Value) -> Result { if let Some((_, flag, reason)) = REFUSED_DESTS.iter().find(|(name, _, _)| *name == dest) { return Err(format!("{flag} is not available for ad-hoc runs: {reason}")); } - if dest == LOAD_SOURCES_DEST && *value == Value::Bool(true) { - return Err( - "--load is not available for ad-hoc runs: source loads are not part of ad-hoc runs yet" - .into(), - ); - } if !ALLOWED_DESTS.contains(&dest) { return Err(format!( "{} is not available for ad-hoc runs", @@ -183,6 +202,8 @@ fn setting(dest: &str, value: &Value) -> Result { .parse::() .map(|number| json!(number)) .map_err(|_| format!("{} must be a whole number", flag_of(dest))), + Value::Str(text) if dest == VARS_DEST => serde_json::from_str(text) + .map_err(|error| format!("--vars must be valid JSON: {error}")), Value::Str(text) => Ok(json!(text)), _ => Err(format!( "{} is not available for ad-hoc runs", @@ -362,6 +383,28 @@ fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap Result, String> { + let unmanaged: BTreeSet<&str> = publication + .0 + .iter() + .filter(|resource| resource.loader.is_none()) + .map(|resource| resource.key.as_str()) + .collect(); + let sources: BTreeSet = resolved(publication, select, exclude, true)? + .into_iter() + .filter(|key| key.starts_with("source:") && !unmanaged.contains(key.as_str())) + .collect(); + if sources.is_empty() { + return Err("the selectors match no managed sources in the current publication".into()); + } + Ok(sources) +} + /// Sources `sqb freshness` observes: selected or upstream sources, less those the exclusions reach. fn freshness_sources( publication: (&[PlannerResource], &PublishedJobs), diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index dc93f3bcba..4c7a2e269e 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -1,7 +1,29 @@ //! Which commands and flags an ad-hoc run accepts; anything not listed is refused. /// Verbs a person may plan; workers launch only those the control plane lists. -pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "freshness"]; +pub(crate) const ADHOC_VERBS: [&str; 6] = ["build", "test", "seed", "audit", "freshness", "load"]; + +/// The verb that runs only the loaders of managed sources, planned as a build of those sources. +pub(crate) const LOAD_VERB: &str = "load"; + +/// The build settings a load runs with: loaders on, no tests or audits, as `sqb load` does. +pub(crate) const LOAD_BUILD_SETTINGS: [(&str, bool); 3] = [ + (LOAD_SOURCES_DEST, true), + ("run_tests", false), + ("run_audits", false), +]; + +/// The destination `--load` and `--no-load` share. +pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; + +/// The destination `--reload` sets; it implies loading. +pub(crate) const RELOAD_DEST: &str = "reload"; + +/// The destination `--vars` sets; its JSON object becomes the run's `vars` setting. +pub(crate) const VARS_DEST: &str = "vars"; + +/// The converter that reads `--vars` as a JSON object. +pub(crate) const VARS_CONVERTER: &str = "parse_cli_vars"; /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; @@ -30,14 +52,11 @@ pub(crate) const CHANGED_DEST: &str = "changed"; /// The destination `--no-python` clears; only a build with Python nodes resolves them. pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python"; -/// The destination `--load` and `--no-load` share; only `--no-load` is accepted. -pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; - /// Parser destinations that carry selectors, resolved to exact node keys on the server. pub(crate) const SELECTOR_DESTS: [&str; 3] = ["select", "exclude", "positional_select"]; /// Destinations an ad-hoc run may set; the control plane gates history rewrites and warehouses. -pub(crate) const ALLOWED_DESTS: [&str; 29] = [ +pub(crate) const ALLOWED_DESTS: [&str; 31] = [ "max_microbatches", "changed", "reach", @@ -67,6 +86,8 @@ pub(crate) const ALLOWED_DESTS: [&str; 29] = [ "allow_retention_decrease", "warehouse", "load_sources", + "reload", + "vars", ]; /// Parser converters that take whole numbers; the table names them as Python does. @@ -76,23 +97,17 @@ pub(crate) const INTEGER_CONVERTERS: [&str; 2] = ["int", POSITIVE_INTEGER_CONVER pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; /// Refused destinations: the flag people typed and why an ad-hoc run cannot honour it. -pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 25] = [ +pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 23] = [ ( "select_file", "--select-file", "it reads a file on the worker", ), - ( - "reload", - "--reload", - "source loads are not part of ad-hoc runs yet", - ), ( "allow_missing_migration_origin", "--allow-missing-migration-origin", "it can lose data", ), - ("vars", "--vars", "variables change the published SQL"), ("defer_to", "--defer-to", "it reads another target"), ( "defer_sources_to", diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index df7501c9c1..cdee715583 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -162,9 +162,33 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select expected_plan: Ok((&["model:customers"], json!({}))), }, AdhocPlanTestCase { - description: "variables are refused", - command: &["build", "--vars", "{\"day\": 1}"], - expected_plan: Err("--vars is not available for ad-hoc runs"), + description: "variables become a JSON object setting, as a job's", + command: &[ + "build", + "-s", + "customers", + "--vars", + "{\"day\": 1, \"region\": \"eu\"}", + ], + expected_plan: Ok(( + &["model:customers"], + json!({"vars": {"day": 1, "region": "eu"}}), + )), + }, + AdhocPlanTestCase { + description: "empty variables are no variables", + command: &["build", "-s", "customers", "--vars", "{}"], + expected_plan: Ok((&["model:customers"], json!({}))), + }, + AdhocPlanTestCase { + description: "variables must be an object", + command: &["build", "--vars", "[1]"], + expected_plan: Err("--vars must be a JSON object"), + }, + AdhocPlanTestCase { + description: "variables must be JSON", + command: &["build", "--vars", "{day: 1}"], + expected_plan: Err("--vars must be valid JSON"), }, AdhocPlanTestCase { description: "a selector file is refused", @@ -172,9 +196,14 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select expected_plan: Err("--select-file is not available"), }, AdhocPlanTestCase { - description: "loading sources is refused", - command: &["build", "--load"], - expected_plan: Err("--load is not available"), + description: "--load becomes a setting for the server to gate", + command: &["build", "-s", "customers", "--load"], + expected_plan: Ok((&["model:customers"], json!({"load_sources": true}))), + }, + AdhocPlanTestCase { + description: "--reload becomes a setting for the server to gate", + command: &["build", "-s", "customers", "--reload"], + expected_plan: Ok((&["model:customers"], json!({"reload": true}))), }, AdhocPlanTestCase { description: "another target is refused", @@ -217,9 +246,9 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select expected_plan: Err("returns"), }, AdhocPlanTestCase { - description: "loads are not ad-hoc runs yet", + description: "a load needs managed sources", command: &["load"], - expected_plan: Err("sqb load is not available as an ad-hoc run"), + expected_plan: Err("the selectors match no managed sources"), }, AdhocPlanTestCase { description: "commands that are not runs are refused", @@ -471,6 +500,30 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co command: &["build", "--select", "loader:stage_shipments"], expected_plan: Ok((&["source:stage_shipments"], json!({}))), }, + AdhocPlanTestCase { + description: "a load without selectors runs every managed source as a build without tests or audits", + command: &["load"], + expected_plan: Ok(( + &["source:raw_shipments", "source:stage_shipments"], + json!({"load_sources": true, "run_audits": false, "run_tests": false}), + )), + }, + AdhocPlanTestCase { + description: "a load selects a source by name and leaves the models that read it", + command: &[ + "load", + "--select", + "raw_shipments", + "+shipments", + "--reload", + "--vars", + "{\"day\": 1}", + ], + expected_plan: Ok(( + &["source:raw_shipments", "source:stage_shipments"], + json!({"reload": true, "run_audits": false, "run_tests": false, "vars": {"day": 1}}), + )), + }, AdhocPlanTestCase { description: "test selects only the SQL nodes a tag shared with Python nodes matches", command: &["test", "--select", "tag:orders"], diff --git a/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs b/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs index 3e62a01a4f..171b36d2db 100644 --- a/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs +++ b/crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs @@ -12,7 +12,7 @@ use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; use crate::store::types::Row; -const ACTIONS: [Action; 16] = [ +const ACTIONS: [Action; 18] = [ Action::View, Action::ViewSql, Action::ViewEvidence, @@ -22,6 +22,8 @@ const ACTIONS: [Action; 16] = [ Action::RunRewriteHistory, Action::RunPrioritise, Action::RunWarehouse, + Action::RunVars, + Action::RunLoad, Action::TriggerToggle, Action::SensorResetCursor, Action::RunAdhoc, diff --git a/crates/sqlbuild-orchestration/src/credentials/models.rs b/crates/sqlbuild-orchestration/src/credentials/models.rs index d965dae987..b5b987e0f6 100644 --- a/crates/sqlbuild-orchestration/src/credentials/models.rs +++ b/crates/sqlbuild-orchestration/src/credentials/models.rs @@ -163,6 +163,10 @@ pub enum Action { RunPrioritise, #[serde(rename = "run.warehouse")] RunWarehouse, + #[serde(rename = "run.vars")] + RunVars, + #[serde(rename = "run.load")] + RunLoad, #[serde(rename = "trigger.toggle")] TriggerToggle, #[serde(rename = "sensor.reset_cursor")] diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index d94c0d4073..19e399be8a 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -64,7 +64,13 @@ pub(crate) async fn frozen_adhoc_request( Ok(plan) => plan, Err(refusal) => return Ok((transaction, Err(refusal))), }; - let Some(command) = RunCommand::of_verb(&plan.verb) else { + // A load runs as a build of its managed sources, so the worker's verified build path enforces its write set. + let verb = if plan.verb == "load" { + "build" + } else { + plan.verb.as_str() + }; + let Some(command) = RunCommand::of_verb(verb) else { let refusal = format!("workers do not run sqb {}", plan.verb); return Ok((transaction, Err(refusal))); }; @@ -85,7 +91,7 @@ pub(crate) async fn frozen_adhoc_request( .and_then(|row| row["image_id"].as_str()) .map(str::to_owned); let write_set: BTreeSet = match plan.verb.as_str() { - "build" => writers, + "build" | "load" => writers, "seed" => writers .into_iter() .filter(|key| key.starts_with("seed:")) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs index 16bdf00f11..ddb444f8e5 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -21,6 +21,9 @@ const CURSOR_SETTINGS: [&str; 6] = [ "end_cursor_int", ]; +/// Settings that ask for source loads beyond a plain build's automatic ones; the `load` verb sets `load_sources`. +const EXPLICIT_LOAD_SETTINGS: [&str; 2] = ["load_sources", "reload"]; + /// Confirmations of destructive changes an ad-hoc command may set; a published job's authored settings keep job.run. const ADHOC_DESTRUCTIVE_SETTINGS: [&str; 4] = [ "allow_snapshot_full_refresh", @@ -81,10 +84,13 @@ pub(crate) fn execution_scopes( if launches && adhoc && request.settings.contains_key("warehouse") { actions.push(Action::RunWarehouse); } + if launches && adhoc && request.settings.contains_key("vars") { + actions.push(Action::RunVars); + } if launches && request.priority > 0 { actions.push(Action::RunPrioritise); } - actions + let mut scopes: Vec = actions .into_iter() .map(|action| PermissionScope { action, @@ -92,6 +98,36 @@ pub(crate) fn execution_scopes( target: Some(request.target.clone()), nodes: nodes.clone(), }) + .collect(); + let loaded = explicitly_loaded(request); + if launches && adhoc && !loaded.is_empty() { + scopes.push(PermissionScope { + action: Action::RunLoad, + location_id: Some(request.location_id), + target: Some(request.target.clone()), + nodes: loaded, + }); + } + scopes +} + +/// The sources an ad-hoc `--load`, `--reload` or `load` writes; a plain build's automatic loads need only `run.adhoc`. +fn explicitly_loaded(request: &RunRequest) -> BTreeSet { + let explicit = EXPLICIT_LOAD_SETTINGS.iter().any(|name| { + request + .settings + .get(*name) + .and_then(serde_json::Value::as_bool) + == Some(true) + }); + if !explicit { + return BTreeSet::new(); + } + request + .write_set + .iter() + .filter(|key| key.starts_with("source:")) + .cloned() .collect() } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index a3af046bbd..b4eb2ad090 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -33,7 +33,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ expected_launch: Ok(true), }, AdhocCommandTestCase { - description: "a build claims the managed source it loads for a selected model and its loader chain", + description: "a Developer's plain build loads the managed source a selected model reads, and its loader chain, under run.adhoc alone", person: AdhocPerson::Developer, command: &["build", "model:customers"], priority: 0, @@ -99,6 +99,72 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ "permissions": [["run.adhoc", true], ["run.warehouse", false]], "error": null, "launchable": false}), expected_launch: Err(403), }, + AdhocCommandTestCase { + description: "Developer lacks run.load for an explicit --load of the sources a model reads", + person: AdhocPerson::Developer, + command: &["build", "model:customers", "--load"], + priority: 0, + reason: "Refresh customers", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.accounts", "source:shop.customers"], + "permissions": [["run.adhoc", true], ["run.load", false]], "error": null, "launchable": false}), + expected_launch: Err(403), + }, + AdhocCommandTestCase { + description: "Developer lacks run.load for --reload", + person: AdhocPerson::Developer, + command: &["build", "model:customers", "--reload"], + priority: 0, + reason: "Reload customers", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.accounts", "source:shop.customers"], + "permissions": [["run.adhoc", true], ["run.load", false]], "error": null, "launchable": false}), + expected_launch: Err(403), + }, + AdhocCommandTestCase { + description: "Developer lacks run.load for the load verb", + person: AdhocPerson::Developer, + command: &["load", "source:shop.customers"], + priority: 0, + reason: "Load customers", + expected_check: json!({"selection": ["source:shop.customers"], "write_set": ["source:shop.customers"], + "permissions": [["run.adhoc", true], ["run.load", false]], "error": null, "launchable": false}), + expected_launch: Err(403), + }, + AdhocCommandTestCase { + description: "--load that loads no managed source needs no run.load", + person: AdhocPerson::Developer, + command: &["build", "model:orders", "--load"], + priority: 0, + reason: "Refresh orders", + expected_check: json!({"selection": ["model:orders"], "write_set": ["model:orders"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, + AdhocCommandTestCase { + description: "Developer lacks run.vars", + person: AdhocPerson::Developer, + command: &["build", "model:orders", "--vars", "{\"day\": 1}"], + priority: 0, + reason: "Refresh orders for one day", + expected_check: json!({"selection": ["model:orders"], "write_set": ["model:orders"], + "permissions": [["run.adhoc", true], ["run.vars", false]], "error": null, "launchable": false}), + expected_launch: Err(403), + }, + AdhocCommandTestCase { + description: "Admin loads with variables, needing run.vars and run.load on the loaded source only", + person: AdhocPerson::Admin, + command: &[ + "load", + "source:shop.customers", + "--reload", + "--vars", + "{\"day\": 1}", + ], + priority: 0, + reason: "Reload customers for one day", + expected_check: json!({"selection": ["source:shop.customers"], "write_set": ["source:shop.customers"], + "permissions": [["run.adhoc", true], ["run.vars", true], ["run.load", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, AdhocCommandTestCase { description: "Developer lacks run.prioritise alone", person: AdhocPerson::Developer, @@ -159,11 +225,11 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ AdhocCommandTestCase { description: "a refused flag is refused before any permission", person: AdhocPerson::Admin, - command: &["build", "--vars"], + command: &["build", "--debug"], priority: 0, reason: "Refresh orders", expected_check: json!({"selection": [], "write_set": [], "permissions": [], - "error": "--vars is not available for ad-hoc runs", "launchable": false}), + "error": "--debug is not available for ad-hoc runs", "launchable": false}), expected_launch: Err(400), }, AdhocCommandTestCase { @@ -266,14 +332,22 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ ], "{backend:?}: the worker receives the exact resolved selection of the pinned publication" ); - let tested = test_cases - .iter() - .position(|test_case| test_case.command[0] == "test") - .expect("a test launch"); + let position = |verb: &str| { + test_cases + .iter() + .position(|test_case| { + test_case.command[0] == verb && test_case.expected_launch.is_ok() + }) + .expect("a launch of the verb") + }; let commands = rows( store, - "SELECT request_json FROM sqb_runs WHERE run_id=$1 OR run_id=$2 ORDER BY run_id", - vec![Parameter::Id(run_ids[0]), Parameter::Id(run_ids[tested])], + "SELECT request_json FROM sqb_runs WHERE run_id IN ($1, $2, $3) ORDER BY run_id", + vec![ + Parameter::Id(run_ids[0]), + Parameter::Id(run_ids[position("load")]), + Parameter::Id(run_ids[position("test")]), + ], ) .await .iter() @@ -281,8 +355,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ .collect::>>(); assert_eq!( commands, - vec![None, Some(json!("test"))], - "{backend:?}: the worker receives the command, and a build request is serialized as before" + vec![None, None, Some(json!("test"))], + "{backend:?}: the worker receives the command, a load runs as a build, and a build request is serialized as before" ); let audit = rows( store, @@ -306,6 +380,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ json!(["build", {}, "Refresh customers"]), json!(["build", {}, "Reload customers"]), json!(["build", {"load_sources": false}, "Refresh customers only"]), + json!(["build", {"load_sources": true}, "Refresh orders"]), + json!(["load", {"reload": true, "vars": {"day": 1}}, "Reload customers for one day"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), ], diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index cbc5219fa8..b4ee151d20 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -194,6 +194,22 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { .insert("load_sources".into(), serde_json::json!(false)); continue; } + "--load" | "--reload" => { + let name = if token == "--load" { + "load_sources" + } else { + "reload" + }; + plan.settings.insert(name.into(), serde_json::json!(true)); + continue; + } + "--vars" => { + let text = tokens.next().ok_or("--vars needs a JSON object")?; + let vars: serde_json::Value = + serde_json::from_str(text).map_err(|error| error.to_string())?; + plan.settings.insert("vars".into(), vars); + continue; + } "--warehouse" => { let name = tokens.next().ok_or("--warehouse needs a name")?; plan.settings @@ -223,6 +239,11 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { .collect(); } plan.selection.retain(|key| !excluded.contains(key)); + // As the CLI's planner: a load is a build of its sources with loading on. + if plan.verb == "load" && !plan.settings.contains_key("reload") { + plan.settings + .insert("load_sources".into(), serde_json::json!(true)); + } Ok(plan) } } From 5d00d6bfac178bf72453c60b5f8af53182814c0e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:35:19 +0000 Subject: [PATCH 2/5] feat(server): record compiled hashes for ad-hoc variables and gate loads in the UI --- clients/typescript/src/schema.d.ts | 8 +- .../src/adhoc/_helpers/planner.rs | 1 - .../src/runs/_helpers/lifecycle/changes.rs | 5 +- .../src/runs/_helpers/lifecycle/projection.rs | 3 +- .../src/runs/_helpers/requests/adhoc.rs | 11 +- .../src/runs/constants.rs | 3 + .../sqlbuild-orchestration/src/runs/models.rs | 5 + .../src/snapshots/main/mod.rs | 1 + .../src/snapshots/main/node_hashes.rs | 18 ++ .../src/snapshots/main/verify_execution.rs | 9 +- .../_helpers/location_command/command.rs | 47 +++-- .../_helpers/location_command/worker.rs | 39 +++- .../sqlbuild/references/docs/cli/server.md | 17 +- .../publication/_test_types.py | 17 ++ .../publication/test_adhoc.py | 173 ++++++++++++++++- ui/scripts/extract-command-table.mjs | 18 +- ui/src/lib/api/mock/access.ts | 2 + ui/src/lib/api/mock/operate-store.ts | 14 +- ui/src/lib/api/types.ts | 2 + ui/src/lib/api/types/operate.ts | 2 +- ui/src/lib/platform/permission-catalog.ts | 2 + ui/src/lib/run/command-table.generated.json | 179 ++++++++++++++++++ website/src/content/docs/docs/cli/server.mdx | 17 +- 23 files changed, 531 insertions(+), 62 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index fd2e5a038d..be0db5ce1e 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1410,7 +1410,7 @@ export interface components { * @description Closed permission vocabulary; unknown actions fail deserialization. * @enum {string} */ - Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view"; + Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "run.vars" | "run.load" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view"; /** @description What launching the command would run and whether this person may launch it. */ AdhocCheck: { /** @description Why the command cannot run regardless of permissions, such as a refused flag or an unknown selector. */ @@ -2785,6 +2785,12 @@ export interface components { kind: "check_finished"; node_key: string; outcome: components["schemas"]["NodeOutcome"]; + } | { + /** @enum {string} */ + kind: "compiled"; + node_hashes: { + [key: string]: string; + }; } | { /** @enum {string} */ kind: "unresponsive"; diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 255154d2c4..6c4454814a 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -76,7 +76,6 @@ impl CommandPlanner for CliPlanner { for (dest, value) in LOAD_BUILD_SETTINGS { settings.insert(dest.to_owned(), json!(value)); } - // `--reload` implies loading, and a build refuses it together with `--load`. if settings.get(RELOAD_DEST) == Some(&Json::Bool(true)) { settings.remove(LOAD_SOURCES_DEST); } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs index c71085d635..3b21a54d8b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs @@ -70,6 +70,7 @@ pub(crate) fn validate_authority( | RunChange::NodeFinished { .. } | RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } + | RunChange::Compiled { .. } ), RunAuthority::Worker { run_id, claim_id } => { run_id == request.run_id @@ -81,6 +82,7 @@ pub(crate) fn validate_authority( | RunChange::NodeFinished { .. } | RunChange::CheckStarted { .. } | RunChange::CheckFinished { .. } + | RunChange::Compiled { .. } | RunChange::Recovered | RunChange::Finished { outcome: RunOutcome::Success @@ -130,7 +132,8 @@ pub(crate) fn validate_change( RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } | RunChange::CheckStarted { .. } - | RunChange::CheckFinished { .. } => { + | RunChange::CheckFinished { .. } + | RunChange::Compiled { .. } => { matches!(state.status, Running | Unresponsive) } RunChange::Unresponsive => state.status == Running, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs index da765e4e32..d0de9e4880 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs @@ -58,7 +58,8 @@ pub(crate) async fn project_run_state( match &request.change { RunChange::LaunchFailed { .. } | RunChange::CheckStarted { .. } - | RunChange::CheckFinished { .. } => {} + | RunChange::CheckFinished { .. } + | RunChange::Compiled { .. } => {} RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => { return project_node(transaction, request, &state.node_counts_json).await; } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 19e399be8a..eb02840b6c 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -15,7 +15,7 @@ use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; use crate::runs::_helpers::requests::run_id::refuse_derived_run_id; -use crate::runs::constants::ADHOC_AUDIT_TAG; +use crate::runs::constants::{ADHOC_AUDIT_TAG, LOAD_VERB}; use crate::runs::models::{ AdhocCheckRequest, AdhocPermission, AdhocRunRequest, CommandPlan, PlannerResource, RunCommand, RunKind, RunOrigin, RunRequest, @@ -64,13 +64,12 @@ pub(crate) async fn frozen_adhoc_request( Ok(plan) => plan, Err(refusal) => return Ok((transaction, Err(refusal))), }; - // A load runs as a build of its managed sources, so the worker's verified build path enforces its write set. - let verb = if plan.verb == "load" { - "build" + let command = if plan.verb == LOAD_VERB { + Some(RunCommand::Build) } else { - plan.verb.as_str() + RunCommand::of_verb(&plan.verb) }; - let Some(command) = RunCommand::of_verb(verb) else { + let Some(command) = command else { let refusal = format!("workers do not run sqb {}", plan.verb); return Ok((transaction, Err(refusal))); }; diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 3a18b7d850..944dfe3268 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -40,3 +40,6 @@ pub(crate) const RUN_LOG_RETENTION_BATCH: usize = 10; pub(crate) const RUN_LOG_DIGEST_CHUNK: usize = 500; /// The UUID version of the run IDs the server derives from other IDs, which people may not supply. pub(crate) const DERIVED_RUN_ID_VERSION: usize = 5; + +/// A load runs as a build of its managed sources, so the worker's verified build path enforces its write set. +pub(crate) const LOAD_VERB: &str = "load"; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 169212f1bb..eaf0ab25fb 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -565,6 +565,10 @@ pub enum RunChange { node_key: String, outcome: NodeOutcome, }, + /// A run with variables compiles them on the worker; this records each run node's resulting hash. + Compiled { + node_hashes: BTreeMap, + }, Unresponsive, Recovered, CancelRequested { @@ -589,6 +593,7 @@ impl RunChange { Self::NodeFinished { .. } => "run_node_finished", Self::CheckStarted { .. } => "run_check_started", Self::CheckFinished { .. } => "run_check_finished", + Self::Compiled { .. } => "run_compiled", Self::Unresponsive => "run_unresponsive", Self::Recovered => "run_recovered", Self::CancelRequested { .. } => "run_cancel_requested", diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs index c06e3c034a..43ec2c5452 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod holds_on_current_nodes; pub(crate) mod job_write_set; pub mod list_person_snapshots; pub mod next_remote_generation; +pub mod node_hashes; pub(crate) mod prune_snapshots; pub(crate) mod publish_development_snapshot; pub mod publish_remote_snapshot; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs b/crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs new file mode 100644 index 0000000000..ff0e8cbb1f --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/main/node_hashes.rs @@ -0,0 +1,18 @@ +//! The content hash of each compiled node, as a publication pins it. + +use std::collections::BTreeMap; + +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; + +use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::models::SnapshotInput; +use crate::store::errors::ControlError; + +/// Each node's hash over its execution content, the same hash a publication records for it. +pub fn node_hashes(input: &SnapshotInput) -> Result, ControlError> { + input + .nodes + .iter() + .map(|(key, node)| Ok((key.clone(), sha256_hex(&node_content(node)?.definition)))) + .collect() +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs index d4d2c00236..f4a305b5ba 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/verify_execution.rs @@ -1,9 +1,7 @@ //! Refuse execution when a worker's local compilation differs from the pinned publication. -use sqlbuild_recall::records::main::sha256_hex::sha256_hex; - use crate::runs::models::RunExecution; -use crate::snapshots::_helpers::content::node_content; +use crate::snapshots::main::node_hashes::node_hashes; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -38,9 +36,8 @@ pub fn verify_execution( "worker node set differs from the pinned snapshot".into(), )); } - for (key, node) in &input.nodes { - let hash = sha256_hex(&node_content(node)?.definition); - if expected.node_hashes.get(key) != Some(&hash) { + for (key, hash) in node_hashes(input)? { + if expected.node_hashes.get(&key) != Some(&hash) { return Err(ControlError::Invalid( "worker node content differs from the pinned snapshot".into(), )); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 1d6c1c9bee..5f8cb953e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -114,25 +114,39 @@ fn compile_location<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, input: &SnapshotInput, -) -> PyResult<(NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys)> { +) -> PyResult<(NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys)> { status(py, "Discovering location project...")?; - let ProjectAdapter { - project_dir, - discovered, - adapter, - } = project_adapter( + let project = project_adapter( py, Some(arguments.value("project_dir")?), ProjectInputs::Discovered, )?; status(py, "Location project discovered. Compiling snapshot...")?; - let options = new_record_with( - py, - RecordId::CompilePipelineOptions, - &[("selected_target", arguments.value("selected_target")?)], - )?; + let (snapshot, blocked) = compile_snapshot(py, arguments, (input, &project), None)?; + status(py, "Location snapshot compiled.")?; + Ok((snapshot, project, blocked)) +} + +/// Compile the discovered project into a snapshot, with a run's variables when it carries them. +pub(super) fn compile_snapshot<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + (input, project): (&SnapshotInput, &ProjectAdapter<'py>), + vars: Option<&Bound<'py, PyAny>>, +) -> PyResult<(NativeOrchestrationSnapshot, NodeKeys)> { + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project; + let mut options: Vec<(&str, Bound<'py, PyAny>)> = + vec![("selected_target", arguments.value("selected_target")?)]; + if let Some(vars) = vars { + options.push(("cli_vars", vars.clone())); + } + let options = new_record_with(py, RecordId::CompilePipelineOptions, &options)?; let compiled = compiled_phase( - (&discovered, &adapter), + (discovered, adapter), &options, &py.None().into_bound(py), None, @@ -152,8 +166,7 @@ fn compile_location<'py>( .then(Vec::new), ), )?; - status(py, "Location snapshot compiled.")?; - Ok((snapshot, discovered, blocked_nodes(&project)?)) + Ok((snapshot, blocked_nodes(&project)?)) } pub(super) fn location_arguments<'py>( @@ -288,11 +301,11 @@ pub(crate) fn run_location<'py>( arguments.value("run_id")?.extract::()?, arguments.value("claim_id")?.extract::()?, ); - let (snapshot, discovered, blocked) = compile_location(py, &arguments, &input)?; + let (snapshot, project, blocked) = compile_location(py, &arguments, &input)?; return execute_worker( py, - (&snapshot, discovered, blocked), - &arguments, + (&snapshot, project, blocked), + (&arguments, &input), (&bearer, identity, recall), ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 0d4d8157c5..14323410d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -15,10 +15,13 @@ use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::{ - RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, + RunChange, RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, }; +use sqlbuild_orchestration::snapshots::main::node_hashes::node_hashes; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::audit_command::command::run_audit; +use crate::bindings::_helpers::boundary::config_values::json_value_to_python; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, within}; use crate::bindings::_helpers::build_command::command::run_build; use crate::bindings::_helpers::cli::argv::parse_cli_invocation; @@ -34,11 +37,13 @@ use crate::bindings::_helpers::compiled_project::snapshot_worker::NativeOrchestr use crate::bindings::_helpers::compiled_views::project_reads::sql_tests; use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; +use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::{ CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::native_host::host::entry; +use crate::bindings::_helpers::query_command::project::ProjectAdapter; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::session::InvocationRecall; use crate::bindings::_helpers::records::fields::{field, replaced}; @@ -302,12 +307,39 @@ fn run_cancellable<'py>( result } +/// Compile the verified project again with an ad-hoc run's variables and record its nodes' hashes on the run. +fn record_compiled_vars<'py>( + py: Python<'py>, + (owner, frozen): (&Bound<'py, NativeOrchestrationWorker>, &RunRequest), + (arguments, input, project): (&KeywordArguments<'py>, &SnapshotInput, &ProjectAdapter<'py>), +) -> PyResult<()> { + let Some(vars) = frozen + .settings + .get("vars") + .filter(|_| frozen.kind == RunKind::Adhoc) + else { + return Ok(()); + }; + write_stderr(py, "Compiling with the run's variables...\n")?; + let vars = json_value_to_python(py, vars)?; + let (compiled, _) = compile_snapshot(py, arguments, (input, project), Some(&vars))?; + let node_hashes: BTreeMap = node_hashes(&compiled.input) + .map_err(|error| PyValueError::new_err(error.to_string()))? + .into_iter() + .filter(|(key, _)| frozen.selection.contains(key) || frozen.write_set.contains(key)) + .collect(); + owner + .get() + .change(py, "compiled", RunChange::Compiled { node_hashes }) +} + pub(super) fn execute_worker<'py>( py: Python<'py>, - (snapshot, discovered, blocked): (&NativeOrchestrationSnapshot, Bound<'py, PyAny>, NodeKeys), - arguments: &KeywordArguments<'py>, + (snapshot, project, blocked): (&NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys), + (arguments, input): (&KeywordArguments<'py>, &SnapshotInput), (bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>), ) -> PyResult { + let discovered = project.discovered.clone(); write_stderr(py, "Verifying frozen run inputs...\n")?; let owner = Bound::new( py, @@ -344,6 +376,7 @@ pub(super) fn execute_worker<'py>( write_stderr(py, &format!("Run refused: {reason}\n"))?; return Err(PyValueError::new_err(reason)); } + record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; let (request, scope) = command_request(py, &discovered, arguments, &owner)?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; within(&scope, |_| { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index e0beb2e323..1a9af4c416 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -501,9 +501,12 @@ used. The target is always the one in the request. | `audit` | The audits of the selected nodes, against what is already built | | `seed` | The selected seeds; other selected nodes are left out | | `freshness` | The selected sources and every source upstream of the selection | +| `load` | The loaders of the selected managed sources, as a build of those sources without tests or audits | | Any other verb | Refused | -Only `build` and `seed` write, so `test`, `audit` and `freshness` runs never wait for other runs. +Only `build`, `seed` and `load` write, so `test`, `audit` and `freshness` runs never wait for other +runs. A `build` loads the managed sources its models read, as `sqb build` does, under `run.adhoc` +alone; `--no-load` turns that off. A `test` or `audit` run reports each selected node with the worst result of its tests or audits. Selectors use the CLI grammar: names, `seed:`, `source:`, `tag:`, `path:` and `test:`, with `+` @@ -515,6 +518,8 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--full-refresh`, `--start-cursor`, `--end-cursor`, `--start-cursor-ts`, `--end-cursor-ts`, `--start-cursor-int`, `--end-cursor-int` | `build` | `run.rewrite_history` | | `--allow-snapshot-full-refresh`, `--allow-snapshot-schema-change`, `--allow-table-type-downgrade`, `--allow-retention-decrease` | `build` | `run.rewrite_history` | | `--warehouse` | All | `run.warehouse`, and the warehouse configured for the target on the location | +| `--vars` | All | `run.vars` | +| `--load`, `--reload`, and the `load` verb | `build`, `load` | `run.load` on the sources the run loads | | `--max-microbatches`, `--no-load`, `--no-tests`, `--no-audits`, `--no-python`, `--fail-fast`, `--selection-diagnostics`, `--verbose` | `build` | `run.adhoc` | | `--concurrency` | `build`, `test`, `seed`, `audit` | `run.adhoc` | | `--test-timeout` | `build`, `test` | `run.adhoc` | @@ -522,14 +527,12 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--state`, `--fail-on-error`, `--fail-on-stale` | `freshness` | `run.adhoc` | A priority above 0 also needs `run.prioritise`. Among built-in roles only Admin grants -`run.rewrite_history`, `run.warehouse` and `run.prioritise`. Every other flag is refused with -its reason: +`run.rewrite_history`, `run.warehouse`, `run.vars`, `run.load` and `run.prioritise`. Every other +flag is refused with its reason: | Refused flags | Reason | | --- | --- | -| `--vars` | Variables change the published SQL | | `--defer-to`, `--defer-sources-to`, `--defer-clone-from` | They read another target | -| `--load`, `--reload` | Source loads are not part of ad-hoc runs | | `--allow-missing-migration-origin` | It can lose data | | `--no-sql-analysis`, `--no-cache` | Workers verify the published SQL and manage their own cache | | `--debug` | It prints executed SQL | @@ -544,6 +547,10 @@ the reason `the requested warehouse is not configured for this target on this lo warehouse statement ran`, which the run page shows. A job's own `warehouse` setting is not checked this way. +A run with `--vars` is checked against the publication like any other run. The worker then +compiles the project again with the variables and records the compiled hash of each of the run's +nodes on the run before any statement runs. + ## Plan builds Anyone with `view` on every node of a target can see what a `build` command would do before it diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index de0a584346..904cdb86d1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -430,6 +430,23 @@ class AdhocRunTestCase: expected_worker_log: str = "" +@dataclass(frozen=True) +class AdhocLoadVarsTestCase: + """An ad-hoc load or run with variables, who may launch it, and what the worker did.""" + + description: str + person: Literal["admin", "developer"] + command: list[str] + expected_permissions: list[list[Any]] + # The HTTP status a refused launch returns; None when it launches and succeeds. + expected_refusal: int | None + expected_inventory_tables: int + day_query: str = "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'daily_customers'" + expected_day_rows: list[tuple[int]] = field(default_factory=lambda: [(0,)]) + # One entry per recorded compile: whether its hash differs from the published one. + expected_compiled: list[bool] = field(default_factory=list) + + @dataclass(frozen=True) class ChangedPlanPreviewTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index b7247fd0a6..d269fe16c1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -10,11 +10,13 @@ import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + AdhocLoadVarsTestCase, AdhocRunTestCase, AdhocVerbTestCase, WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + LOADED_ORDER_FILES, duckdb_rows, scheduled_worker, security_audit_rows, @@ -150,14 +152,14 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ body=scope | { "run_id": str(uuid4()), - "command": ["build", "--vars", "{}"], - "reason": "Try variables", + "command": ["build", "--debug"], + "reason": "Try debug output", "priority": 0, }, session=session, ) assert refused.value.code == 400 - assert "--vars is not available for ad-hoc runs" in refused.value.read().decode() + assert "--debug is not available for ad-hoc runs" in refused.value.read().decode() run_id: str = str(uuid4()) launch: dict[str, Any] = scope | { "run_id": run_id, @@ -379,3 +381,168 @@ def launch(command: list[str]) -> tuple[dict[str, Any], str, dict[str, Any]]: ) for name, text in test_case.rewrites.items(): assert (worker.project / name).read_text() == text, "the checkout changed under the run" + + +INVENTORY_TABLES: str = ( + "SELECT count(*) FROM information_schema.tables WHERE table_name='raw_inventory'" +) + + +@pytest.mark.parametrize( + "test_case", + [ + AdhocLoadVarsTestCase( + description="a developer's plain build of a managed source loads it under run.adhoc", + person="developer", + command=["build", "--select", "raw_inventory"], + expected_permissions=[["run.adhoc", True]], + expected_refusal=None, + expected_inventory_tables=1, + ), + AdhocLoadVarsTestCase( + description="a developer may not load on request", + person="developer", + command=["load", "--select", "raw_inventory"], + expected_permissions=[["run.adhoc", True], ["run.load", False]], + expected_refusal=403, + expected_inventory_tables=0, + ), + AdhocLoadVarsTestCase( + description="a developer may not reload", + person="developer", + command=["build", "--select", "raw_inventory", "--reload"], + expected_permissions=[["run.adhoc", True], ["run.load", False]], + expected_refusal=403, + expected_inventory_tables=0, + ), + AdhocLoadVarsTestCase( + description="an admin's load runs only the selected source's loader", + person="admin", + command=["load", "--select", "raw_inventory"], + expected_permissions=[["run.adhoc", True], ["run.load", True]], + expected_refusal=None, + expected_inventory_tables=1, + ), + AdhocLoadVarsTestCase( + description="a developer may not set variables", + person="developer", + command=["build", "--select", "daily_customers", "--vars", '{"day": 7}'], + expected_permissions=[["run.adhoc", True], ["run.vars", False]], + expected_refusal=403, + expected_inventory_tables=0, + ), + AdhocLoadVarsTestCase( + description="an admin's variables compile on the worker and their hashes are recorded", + person="admin", + command=["build", "--select", "daily_customers", "--vars", '{"day": 7}'], + expected_permissions=[["run.adhoc", True], ["run.vars", True]], + expected_refusal=None, + expected_inventory_tables=0, + day_query="SELECT day FROM daily_customers", + expected_day_rows=[(7,)], + expected_compiled=[True], + ), + ], + ids=lambda case: case.description, +) +def test_given_ad_hoc_loads_and_variables_when_launched_then_only_permitted_people_run_them( + test_case: AdhocLoadVarsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + worker_case: WorkerExecutionTestCase = WorkerExecutionTestCase( + description=test_case.description, + model_sql="SELECT 5 AS amount", + warehouse_query="SELECT amount FROM orders", + expected_status="success", + expected_rows=[(5,)], + files=LOADED_ORDER_FILES + | { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + '[vars]\nday = "0"\n' + ), + "models/daily_customers.sql": ( + "MODEL (description 'Customers of one day.', materialized table);\n" + "SELECT @@day AS day\n" + ), + }, + ) + with scheduled_worker( + tmp_path, + monkeypatch, + worker_case, + server_arguments=("--password-origin", "https://control.example.com"), + ) as worker: + scheduled: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert scheduled["status"] == "success", worker.log.read_text() + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + developer: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-developer" + ) + sessions: dict[str, tuple[str, str]] = { + "admin": person_login(worker.api, admin["name"]), + "developer": person_login(worker.api, developer["name"]), + } + assert person_request( + worker.api, + "/api/admin/roles", + body={ + "principal_id": developer["principal_id"], + "role": "developer", + "location_id": worker.api.location_id, + "target": "development", + }, + session=sessions["admin"], + ) == {"applied": True} + session: tuple[str, str] = sessions[test_case.person] + scope: dict[str, Any] = {"location_id": worker.api.location_id, "target": "development"} + check: dict[str, Any] = person_request( + worker.api, + "/api/control/runs/adhoc/check", + body=scope | {"command": test_case.command, "priority": 0}, + session=session, + ) + assert [ + [permission["action"], permission["allowed"]] for permission in check["permissions"] + ] == test_case.expected_permissions + run_id: str = str(uuid4()) + launch: dict[str, Any] = scope | { + "run_id": run_id, + "command": test_case.command, + "reason": "Load or compile on request", + "priority": 0, + } + try: + refusal: int | None = None + assert person_request( + worker.api, "/api/control/runs/adhoc", body=launch, session=session + ) == {"applied": True} + completed: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, run_id=run_id + ) + assert completed["status"] == "success", worker.log.read_text() + except HTTPError as error: + refusal = error.code + assert refusal == test_case.expected_refusal + assert duckdb_rows(worker.project / "warehouse.duckdb", INVENTORY_TABLES) == [ + (test_case.expected_inventory_tables,) + ] + assert ( + duckdb_rows(worker.project / "warehouse.duckdb", test_case.day_query) + == test_case.expected_day_rows + ) + compiled: list[tuple[Any, ...]] = sqlite_rows( + worker.control_store, + "SELECT json_extract(payload, '$.change.change.node_hashes.\"model:daily_customers\"'), " + "(SELECT node_hash FROM sqb_snapshot_nodes WHERE node_key='model:daily_customers' " + "LIMIT 1) FROM recall_events " + f"WHERE event_type='run_compiled' AND run_id='{run_id}'", + ) + assert [hashes[0] != hashes[1] for hashes in compiled] == test_case.expected_compiled, ( + "the worker records the hash its variables compiled, not the published one" + ) diff --git a/ui/scripts/extract-command-table.mjs b/ui/scripts/extract-command-table.mjs index 14de1341a4..7142f11b58 100644 --- a/ui/scripts/extract-command-table.mjs +++ b/ui/scripts/extract-command-table.mjs @@ -25,12 +25,10 @@ const VERB_DESCRIPTIONS = { test: 'Run unit and scenario tests for the selection', seed: 'Load seed files', audit: 'Evaluate audits against what is already built', - freshness: 'Check source freshness' + freshness: 'Check source freshness', + load: 'Run the loaders of managed sources' }; -/** Flags the planner refuses although their destination is accepted: `--no-load` shares `--load`'s. */ -const REFUSED_FLAGS = ['--load']; - /** Descriptions for flags the table leaves undocumented. */ const DESCRIPTIONS = { '--select': 'Selector expression, e.g. +fact_orders, tag:finance, path:models/marts/', @@ -68,8 +66,12 @@ const ACCEPTED = new Set([...rustStrings(ALLOWLIST, 'ALLOWED_DESTS'), ...rustStr /** Destinations whose setting needs a permission beyond run.adhoc, as the control plane decides it. */ const REQUIRES = new Map([ ...['full_refresh', ...rustStrings(GATES, 'CURSOR_SETTINGS'), ...rustStrings(GATES, 'ADHOC_DESTRUCTIVE_SETTINGS')].map((dest) => [dest, 'run.rewrite_history']), - ['warehouse', 'run.warehouse'] + ['warehouse', 'run.warehouse'], + ['vars', 'run.vars'], + ['reload', 'run.load'] ]); +/** Flags gated apart from their destination: `--load` needs run.load, the `--no-load` sharing it does not. */ +const FLAG_REQUIRES = new Map([['--load', 'run.load']]); function valueType(action) { const flag = action.options[0]; @@ -95,7 +97,7 @@ function extract(table) { if (!action.options) continue; for (const option of action.options) known.add(option); const flag = action.options.find((o) => o.startsWith('--')); - if (!flag || !ACCEPTED.has(action.dest) || REFUSED_FLAGS.includes(flag)) continue; + if (!flag || !ACCEPTED.has(action.dest)) continue; const help = typeof action.help === 'string' ? action.help : null; if (action.help && typeof action.help === 'object') continue; flags.push({ @@ -104,13 +106,13 @@ function extract(table) { type: valueType(action), choices: action.choices ?? null, repeatable: action.kind === 'extend' || action.kind === 'append', - requires: REQUIRES.get(action.dest) ?? null, + requires: FLAG_REQUIRES.get(flag) ?? REQUIRES.get(action.dest) ?? null, description: help ?? DESCRIPTIONS[flag] ?? '' }); } return { name, description, flags }; }); - for (const flag of [...REFUSED_FLAGS, ...Object.keys(DESCRIPTIONS)]) { + for (const flag of [...FLAG_REQUIRES.keys(), ...Object.keys(DESCRIPTIONS)]) { if (!known.has(flag)) throw new Error(`${flag} is described here but not in the command table for any ad-hoc verb`); } for (const dest of REQUIRES.keys()) { diff --git a/ui/src/lib/api/mock/access.ts b/ui/src/lib/api/mock/access.ts index 3ad8df2bb2..b9c01ba072 100644 --- a/ui/src/lib/api/mock/access.ts +++ b/ui/src/lib/api/mock/access.ts @@ -115,6 +115,8 @@ const PLAIN: Record = { 'run.rewrite_history': 'full refresh or reprocess past data', 'run.prioritise': 'run ahead of the queue', 'run.warehouse': 'choose the warehouse a run uses', + 'run.vars': 'set variables for a run', + 'run.load': 'load sources on request', 'trigger.toggle': 'pause and resume schedules and sensors', 'sensor.reset_cursor': 'reset where a sensor is up to', 'run.adhoc': 'run sqb commands on any selection of nodes', diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index aacde0be9f..d58277ef15 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1111,20 +1111,26 @@ export class MockOperate { ? resolution.nodes.filter((n) => snapshotNode(n)?.kind === 'seed') : parts.verb === 'freshness' ? [...new Set(resolution.nodes.flatMap((n) => [n, ...ancestors(n)]).filter(isSource))] - : resolution.nodes; - if (!selection.length) return refused(parts.verb === 'seed' ? 'the selectors match no seeds in the current publication' : 'no sources are selected or upstream of the selection'); + : parts.verb === 'load' + ? resolution.nodes.filter(isSource) + : resolution.nodes; + const empty = { seed: 'the selectors match no seeds in the current publication', load: 'the selectors match no managed sources in the current publication' } as Partial>; + if (!selection.length) return refused(empty[parts.verb] ?? 'no sources are selected or upstream of the selection'); const writable = selection.filter((n) => !isSource(n)); const writeSet = parts.verb === 'build' ? writable.filter((n) => !(parts.flags['--no-load'] && snapshotNode(n)?.kind === 'loader')) : parts.verb === 'seed' ? writable.filter((n) => snapshotNode(n)?.kind === 'seed') - : []; + : parts.verb === 'load' + ? selection + : []; const scopeTerms = ADHOC_SCOPE[this.#ctx.role()]; const granted = scopeTerms === null ? null : new Set(writeSetOf(scopeTerms.length ? scopeTerms : ['__none__'], [])); const covered = this.#ctx.can('run.adhoc') && (granted === null || selection.every((n) => granted.has(n))); const decide = (action: Capability, allowed: boolean): AdhocPermission => ({ action, allowed, reason: allowed ? null : `missing ${action} permission for this target` }); - const gates = [...new Set(Object.keys(parts.flags).flatMap((f) => flagSpec(parts.verb, f)?.requires ?? []))]; + const verbGate: Capability[] = parts.verb === 'load' ? ['run.load'] : []; + const gates = [...new Set([...Object.keys(parts.flags).flatMap((f) => flagSpec(parts.verb, f)?.requires ?? []), ...verbGate])]; const permissions = [ decide('run.adhoc', covered), ...gates.map((action) => decide(action, this.#ctx.can(action))), diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index 895d746c92..d627752c5f 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -20,6 +20,8 @@ export const CAPABILITIES = [ 'run.rewrite_history', 'run.prioritise', 'run.warehouse', + 'run.vars', + 'run.load', 'trigger.toggle', 'sensor.reset_cursor', 'run.adhoc', diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 0dc7087afa..a9e851e32c 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -537,7 +537,7 @@ export type SnapshotNode = { // ---- ad-hoc runs (the plan page) ---- -export type LaunchVerb = 'build' | 'test' | 'seed' | 'audit' | 'freshness'; +export type LaunchVerb = 'build' | 'test' | 'seed' | 'audit' | 'freshness' | 'load'; /** A command as argument tokens after `sqb`, never shell text; the server resolves its selectors. */ export type AdhocRunRequest = { diff --git a/ui/src/lib/platform/permission-catalog.ts b/ui/src/lib/platform/permission-catalog.ts index ca8d299a86..2b95945255 100644 --- a/ui/src/lib/platform/permission-catalog.ts +++ b/ui/src/lib/platform/permission-catalog.ts @@ -26,6 +26,8 @@ export const PERMISSIONS: Record = { }, 'run.prioritise': { label: 'Put runs ahead of the queue', description: 'Start or rerun a run with a priority above 0, so it is claimed before waiting runs.', risk: 'operate' }, 'run.warehouse': { label: 'Choose a run’s warehouse', description: 'Start an ad-hoc run on another warehouse configured for the target, which can change cost and speed.', risk: 'operate' }, + 'run.vars': { label: 'Set variables for a run', description: 'Start an ad-hoc run with --vars, which compiles different SQL than the published project.', risk: 'changes_data' }, + 'run.load': { label: 'Load sources on request', description: 'Start an ad-hoc load, or a build with --load or --reload, which runs source loaders against the warehouse.', risk: 'changes_data' }, 'trigger.toggle': { label: 'Pause and resume schedules and sensors', description: 'Stop or restart automatic runs.', risk: 'operate' }, 'sensor.reset_cursor': { label: 'Reset where a sensor is up to', description: 'Move a sensor’s position so it looks at events again.', risk: 'operate' }, 'run.adhoc': { label: 'Run ad-hoc commands', description: 'Run sqb build and other commands on any selection of nodes outside a declared job, writing to the warehouse.', risk: 'changes_data' }, diff --git a/ui/src/lib/run/command-table.generated.json b/ui/src/lib/run/command-table.generated.json index 8498e446d6..1faa7a8d94 100644 --- a/ui/src/lib/run/command-table.generated.json +++ b/ui/src/lib/run/command-table.generated.json @@ -104,6 +104,15 @@ "requires": null, "description": "Process at most this many microbatches per incremental model" }, + { + "flag": "--load", + "aliases": [], + "type": "bool", + "choices": null, + "repeatable": false, + "requires": "run.load", + "description": "" + }, { "flag": "--no-load", "aliases": [], @@ -113,6 +122,15 @@ "requires": null, "description": "Skip loaders" }, + { + "flag": "--reload", + "aliases": [], + "type": "bool", + "choices": null, + "repeatable": false, + "requires": "run.load", + "description": "" + }, { "flag": "--fail-fast", "aliases": [], @@ -237,6 +255,15 @@ "requires": null, "description": "Selector expression to leave out" }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, { "flag": "--warehouse", "aliases": [], @@ -308,6 +335,15 @@ "requires": null, "description": "Selector expression to leave out" }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, { "flag": "--warehouse", "aliases": [], @@ -352,6 +388,15 @@ "requires": null, "description": "Selector expression to leave out" }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, { "flag": "--warehouse", "aliases": [], @@ -396,6 +441,15 @@ "requires": null, "description": "Selector expression to leave out" }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, { "flag": "--warehouse", "aliases": [], @@ -458,6 +512,131 @@ "requires": null, "description": "Selector expression to leave out" }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, + { + "flag": "--warehouse", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.warehouse", + "description": "warehouse for this invocation; overrides the target's command-group warehouse and the connection warehouse" + } + ] + }, + { + "name": "load", + "description": "Run the loaders of managed sources", + "flags": [ + { + "flag": "--reload", + "aliases": [], + "type": "bool", + "choices": null, + "repeatable": false, + "requires": "run.load", + "description": "" + }, + { + "flag": "--concurrency", + "aliases": [], + "type": "int", + "choices": null, + "repeatable": false, + "requires": null, + "description": "Nodes to run in parallel within this run" + }, + { + "flag": "--select", + "aliases": [ + "-s" + ], + "type": "selector", + "choices": null, + "repeatable": true, + "requires": null, + "description": "Selector expression, e.g. +fact_orders, tag:finance, path:models/marts/" + }, + { + "flag": "--exclude", + "aliases": [], + "type": "selector", + "choices": null, + "repeatable": true, + "requires": null, + "description": "Selector expression to leave out" + }, + { + "flag": "--start-cursor", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "override the start cursor, typed from the selected models' cursor type" + }, + { + "flag": "--end-cursor", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "override the end cursor, typed from the selected models' cursor type" + }, + { + "flag": "--start-cursor-ts", + "aliases": [], + "type": "timestamp", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "timestamp start cursor override" + }, + { + "flag": "--end-cursor-ts", + "aliases": [], + "type": "timestamp", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "timestamp end cursor override" + }, + { + "flag": "--start-cursor-int", + "aliases": [], + "type": "int", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "integer start cursor override" + }, + { + "flag": "--end-cursor-int", + "aliases": [], + "type": "int", + "choices": null, + "repeatable": false, + "requires": "run.rewrite_history", + "description": "integer end cursor override" + }, + { + "flag": "--vars", + "aliases": [], + "type": "string", + "choices": null, + "repeatable": false, + "requires": "run.vars", + "description": "" + }, { "flag": "--warehouse", "aliases": [], diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 0cb5dc46d5..2536a86525 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -484,9 +484,12 @@ used. The target is always the one in the request. | `audit` | The audits of the selected nodes, against what is already built | | `seed` | The selected seeds; other selected nodes are left out | | `freshness` | The selected sources and every source upstream of the selection | +| `load` | The loaders of the selected managed sources, as a build of those sources without tests or audits | | Any other verb | Refused | -Only `build` and `seed` write, so `test`, `audit` and `freshness` runs never wait for other runs. +Only `build`, `seed` and `load` write, so `test`, `audit` and `freshness` runs never wait for other +runs. A `build` loads the managed sources its models read, as `sqb build` does, under `run.adhoc` +alone; `--no-load` turns that off. A `test` or `audit` run reports each selected node with the worst result of its tests or audits. Selectors use the CLI grammar: names, `seed:`, `source:`, `tag:`, `path:` and `test:`, with `+` @@ -498,6 +501,8 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--full-refresh`, `--start-cursor`, `--end-cursor`, `--start-cursor-ts`, `--end-cursor-ts`, `--start-cursor-int`, `--end-cursor-int` | `build` | `run.rewrite_history` | | `--allow-snapshot-full-refresh`, `--allow-snapshot-schema-change`, `--allow-table-type-downgrade`, `--allow-retention-decrease` | `build` | `run.rewrite_history` | | `--warehouse` | All | `run.warehouse`, and the warehouse configured for the target on the location | +| `--vars` | All | `run.vars` | +| `--load`, `--reload`, and the `load` verb | `build`, `load` | `run.load` on the sources the run loads | | `--max-microbatches`, `--no-load`, `--no-tests`, `--no-audits`, `--no-python`, `--fail-fast`, `--selection-diagnostics`, `--verbose` | `build` | `run.adhoc` | | `--concurrency` | `build`, `test`, `seed`, `audit` | `run.adhoc` | | `--test-timeout` | `build`, `test` | `run.adhoc` | @@ -505,14 +510,12 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--state`, `--fail-on-error`, `--fail-on-stale` | `freshness` | `run.adhoc` | A priority above 0 also needs `run.prioritise`. Among built-in roles only Admin grants -`run.rewrite_history`, `run.warehouse` and `run.prioritise`. Every other flag is refused with -its reason: +`run.rewrite_history`, `run.warehouse`, `run.vars`, `run.load` and `run.prioritise`. Every other +flag is refused with its reason: | Refused flags | Reason | | --- | --- | -| `--vars` | Variables change the published SQL | | `--defer-to`, `--defer-sources-to`, `--defer-clone-from` | They read another target | -| `--load`, `--reload` | Source loads are not part of ad-hoc runs | | `--allow-missing-migration-origin` | It can lose data | | `--no-sql-analysis`, `--no-cache` | Workers verify the published SQL and manage their own cache | | `--debug` | It prints executed SQL | @@ -527,6 +530,10 @@ the reason `the requested warehouse is not configured for this target on this lo warehouse statement ran`, which the run page shows. A job's own `warehouse` setting is not checked this way. +A run with `--vars` is checked against the publication like any other run. The worker then +compiles the project again with the variables and records the compiled hash of each of the run's +nodes on the run before any statement runs. + ## Plan builds Anyone with `view` on every node of a target can see what a `build` command would do before it From 3f60bbb3c9294c773aa082bc02dc6385bc28bdd4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:42:15 +0000 Subject: [PATCH 3/5] fix(ui): describe the --vars, --load and --reload options --- ui/scripts/extract-command-table.mjs | 5 ++++- ui/src/lib/run/command-table.generated.json | 18 +++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ui/scripts/extract-command-table.mjs b/ui/scripts/extract-command-table.mjs index 7142f11b58..bafa50b309 100644 --- a/ui/scripts/extract-command-table.mjs +++ b/ui/scripts/extract-command-table.mjs @@ -38,6 +38,9 @@ const DESCRIPTIONS = { '--no-audits': 'Skip audits during the build', '--max-microbatches': 'Process at most this many microbatches per incremental model', '--no-load': 'Skip loaders', + '--load': 'Load managed sources before building', + '--reload': 'Reload managed sources (loaders get is_reload=True)', + '--vars': 'Project variables as a JSON object, e.g. {"day": "7"}', '--fail-fast': 'Stop at the first failed node', '--full-refresh': 'Rebuild incremental models and snapshots from scratch', '--allow-snapshot-full-refresh': 'Allow a full refresh of snapshot models', @@ -107,7 +110,7 @@ function extract(table) { choices: action.choices ?? null, repeatable: action.kind === 'extend' || action.kind === 'append', requires: FLAG_REQUIRES.get(flag) ?? REQUIRES.get(action.dest) ?? null, - description: help ?? DESCRIPTIONS[flag] ?? '' + description: help || DESCRIPTIONS[flag] || '' }); } return { name, description, flags }; diff --git a/ui/src/lib/run/command-table.generated.json b/ui/src/lib/run/command-table.generated.json index 1faa7a8d94..516150cca6 100644 --- a/ui/src/lib/run/command-table.generated.json +++ b/ui/src/lib/run/command-table.generated.json @@ -111,7 +111,7 @@ "choices": null, "repeatable": false, "requires": "run.load", - "description": "" + "description": "Load managed sources before building" }, { "flag": "--no-load", @@ -129,7 +129,7 @@ "choices": null, "repeatable": false, "requires": "run.load", - "description": "" + "description": "Reload managed sources (loaders get is_reload=True)" }, { "flag": "--fail-fast", @@ -262,7 +262,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", @@ -342,7 +342,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", @@ -395,7 +395,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", @@ -448,7 +448,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", @@ -519,7 +519,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", @@ -543,7 +543,7 @@ "choices": null, "repeatable": false, "requires": "run.load", - "description": "" + "description": "Reload managed sources (loaders get is_reload=True)" }, { "flag": "--concurrency", @@ -635,7 +635,7 @@ "choices": null, "repeatable": false, "requires": "run.vars", - "description": "" + "description": "Project variables as a JSON object, e.g. {\"day\": \"7\"}" }, { "flag": "--warehouse", From 06460489966a8675d964621ee49291ddae43e693 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:51:42 +0000 Subject: [PATCH 4/5] fix(server): plan previews with variables need run.vars; align the load mock with the server --- .../src/runs/_helpers/requests/adhoc.rs | 2 +- .../src/runs/_helpers/requests/permissions.rs | 3 ++- .../src/runs/_helpers/requests/plan.rs | 26 ++++++++++++++++--- .../src/runs/constants.rs | 3 +++ .../tests/api/test_person_plans.rs | 26 +++++++++++++++++-- .../tests/runs/test_adhoc.rs | 5 ++-- .../tests/runs/test_types.rs | 13 +++++++--- .../sqlbuild/references/docs/cli/server.md | 2 +- ui/src/lib/api/mock/operate-store.ts | 18 ++++++++++--- website/src/content/docs/docs/cli/server.mdx | 2 +- 10 files changed, 80 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index eb02840b6c..9569e32123 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -90,7 +90,7 @@ pub(crate) async fn frozen_adhoc_request( .and_then(|row| row["image_id"].as_str()) .map(str::to_owned); let write_set: BTreeSet = match plan.verb.as_str() { - "build" | "load" => writers, + "build" | LOAD_VERB => writers, "seed" => writers .into_iter() .filter(|key| key.starts_with("seed:")) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs index ddb444f8e5..4d10e7f258 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -7,6 +7,7 @@ use uuid::Uuid; use crate::authorization::main::authorize_person::authorize_person; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::job::frozen_group_children; +use crate::runs::constants::VARS_SETTING; use crate::runs::models::{RunKind, RunRequest}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; @@ -84,7 +85,7 @@ pub(crate) fn execution_scopes( if launches && adhoc && request.settings.contains_key("warehouse") { actions.push(Action::RunWarehouse); } - if launches && adhoc && request.settings.contains_key("vars") { + if launches && adhoc && request.settings.contains_key(VARS_SETTING) { actions.push(Action::RunVars); } if launches && request.priority > 0 { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs index 89e1c0bbee..d4779d8a75 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -4,20 +4,21 @@ use chrono::Duration; use uuid::Uuid; use crate::authorization::main::authorize_listing::authorize_listing; -use crate::credentials::models::{AuditContext, DefaultRole}; +use crate::authorization::main::authorize_person::authorize_person; +use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::platform::constants::{LOCATION_SILENCE_SECONDS, PLAN_EXPIRY_SECONDS, PLAN_QUEUE_LIMIT}; use crate::platform::main::enqueue_location_work::enqueue_location_work; use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkRequest, PlanRequest}; use crate::runs::_helpers::requests::adhoc::frozen_adhoc_request; -use crate::runs::constants::{PLAN_IGNORED_SETTING, PLAN_REASON}; +use crate::runs::constants::{LOAD_VERB, PLAN_IGNORED_SETTING, PLAN_REASON, VARS_SETTING}; use crate::runs::models::{AdhocCheckRequest, RunCommand}; use crate::runs::types::CommandPlanner; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; -/// Needs view on every current node; a refused command, a silent location or a full queue queues nothing. +/// Needs view on every current node, and run.vars for variables; a refused command, a silent location or a full queue queues nothing. pub(in crate::runs) async fn request_plan_in( transaction: ControlTransaction, (context, principal_id, planner): (&AuditContext, Uuid, &dyn CommandPlanner), @@ -57,11 +58,28 @@ pub(in crate::runs) async fn request_plan_in( ) .await?; let (verb, frozen) = frozen.map_err(ControlError::Invalid)?; - if frozen.command != RunCommand::Build { + if frozen.command != RunCommand::Build || verb == LOAD_VERB { return Err(ControlError::Invalid(format!( "sqb plan previews builds; sqb {verb} cannot be planned" ))); } + let (transaction, context) = if frozen.settings.contains_key(VARS_SETTING) { + let scope = PermissionScope { + action: Action::RunVars, + location_id: Some(location_id), + target: Some(target.to_owned()), + nodes: frozen.selection.union(&frozen.write_set).cloned().collect(), + }; + authorize_person( + transaction, + (&context, principal_id), + &scope, + DefaultRole::None, + ) + .await? + } else { + (transaction, context) + }; let now = transaction.now; let (transaction, rows) = query(transaction, "SELECT \ diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 944dfe3268..5ba782643b 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -43,3 +43,6 @@ pub(crate) const DERIVED_RUN_ID_VERSION: usize = 5; /// A load runs as a build of its managed sources, so the worker's verified build path enforces its write set. pub(crate) const LOAD_VERB: &str = "load"; + +/// The ad-hoc setting that carries `--vars`. +pub(crate) const VARS_SETTING: &str = "vars"; diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index a774eac04b..d8ee50dc0e 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -123,6 +123,28 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "{description}: only builds are planned", description = test_case.description ); + assert!( + matches!( + ask(store, (&context, viewer), location, &plan_request(&["load", "--select", "customers"])).await, + Err(ControlError::Invalid(message)) if message.contains("cannot be planned") + ), + "{description}: a load is not a build preview", + description = test_case.description + ); + assert!( + matches!( + ask( + store, + (&context, viewer), + location, + &plan_request(&["build", "orders", "--vars", r#"{"day": "7"}"#]) + ) + .await, + Err(ControlError::Denied { .. }) + ), + "{description}: variables change the compile, so planning with them needs run.vars", + description = test_case.description + ); assert!( ask(store, (&context, viewer), location, &first) .await @@ -179,10 +201,10 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo } assert!( matches!( - ask(store, (&context, admin), location, &plan_request(&["build", "products"])).await, + ask(store, (&context, admin), location, &plan_request(&["build", "products", "--vars", r#"{"day": "7"}"#])).await, Err(ControlError::Invalid(message)) if message.contains("already waiting") ), - "{description}: four plans per target", + "{description}: an admin may plan with variables, up to four plans per target", description = test_case.description ); let direct = LocationWorkRequest { diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index b4eb2ad090..f8da807f9d 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -122,7 +122,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ AdhocCommandTestCase { description: "Developer lacks run.load for the load verb", person: AdhocPerson::Developer, - command: &["load", "source:shop.customers"], + command: &["load", "--select", "source:shop.customers"], priority: 0, reason: "Load customers", expected_check: json!({"selection": ["source:shop.customers"], "write_set": ["source:shop.customers"], @@ -154,6 +154,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ person: AdhocPerson::Admin, command: &[ "load", + "--select", "source:shop.customers", "--reload", "--vars", @@ -381,7 +382,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ json!(["build", {}, "Reload customers"]), json!(["build", {"load_sources": false}, "Refresh customers only"]), json!(["build", {"load_sources": true}, "Refresh orders"]), - json!(["load", {"reload": true, "vars": {"day": 1}}, "Reload customers for one day"]), + json!(["load", {"reload": true, "run_audits": false, "run_tests": false, "vars": {"day": 1}}, "Reload customers for one day"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), ], diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index b4ee151d20..fd113a5bca 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -239,10 +239,15 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { .collect(); } plan.selection.retain(|key| !excluded.contains(key)); - // As the CLI's planner: a load is a build of its sources with loading on. - if plan.verb == "load" && !plan.settings.contains_key("reload") { - plan.settings - .insert("load_sources".into(), serde_json::json!(true)); + // As the CLI's planner: a load is a build of its sources with loading on and no tests or audits. + if plan.verb == "load" { + for name in ["run_tests", "run_audits"] { + plan.settings.insert(name.into(), serde_json::json!(false)); + } + if !plan.settings.contains_key("reload") { + plan.settings + .insert("load_sources".into(), serde_json::json!(true)); + } } Ok(plan) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 1a9af4c416..434b80f7be 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -518,7 +518,7 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--full-refresh`, `--start-cursor`, `--end-cursor`, `--start-cursor-ts`, `--end-cursor-ts`, `--start-cursor-int`, `--end-cursor-int` | `build` | `run.rewrite_history` | | `--allow-snapshot-full-refresh`, `--allow-snapshot-schema-change`, `--allow-table-type-downgrade`, `--allow-retention-decrease` | `build` | `run.rewrite_history` | | `--warehouse` | All | `run.warehouse`, and the warehouse configured for the target on the location | -| `--vars` | All | `run.vars` | +| `--vars` | All | `run.vars`, also to preview a plan with them | | `--load`, `--reload`, and the `load` verb | `build`, `load` | `run.load` on the sources the run loads | | `--max-microbatches`, `--no-load`, `--no-tests`, `--no-audits`, `--no-python`, `--fail-fast`, `--selection-diagnostics`, `--verbose` | `build` | `run.adhoc` | | `--concurrency` | `build`, `test`, `seed`, `audit` | `run.adhoc` | diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index d58277ef15..345f5f8c83 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1106,13 +1106,14 @@ export class MockOperate { if (resolution.unknown.length) return refused(`no nodes match selector '${resolution.unknown[0]}'`); if (!resolution.nodes.length) return refused('the selection resolves to no nodes'); const isSource = (n: string) => snapshotNode(n)?.kind === 'source'; + const isManaged = (n: string) => snapshotNode(n)?.kind === 'loader'; const selection = parts.verb === 'seed' ? resolution.nodes.filter((n) => snapshotNode(n)?.kind === 'seed') : parts.verb === 'freshness' ? [...new Set(resolution.nodes.flatMap((n) => [n, ...ancestors(n)]).filter(isSource))] : parts.verb === 'load' - ? resolution.nodes.filter(isSource) + ? resolution.nodes.filter(isManaged) : resolution.nodes; const empty = { seed: 'the selectors match no seeds in the current publication', load: 'the selectors match no managed sources in the current publication' } as Partial>; if (!selection.length) return refused(empty[parts.verb] ?? 'no sources are selected or upstream of the selection'); @@ -1129,8 +1130,17 @@ export class MockOperate { const granted = scopeTerms === null ? null : new Set(writeSetOf(scopeTerms.length ? scopeTerms : ['__none__'], [])); const covered = this.#ctx.can('run.adhoc') && (granted === null || selection.every((n) => granted.has(n))); const decide = (action: Capability, allowed: boolean): AdhocPermission => ({ action, allowed, reason: allowed ? null : `missing ${action} permission for this target` }); - const verbGate: Capability[] = parts.verb === 'load' ? ['run.load'] : []; - const gates = [...new Set([...Object.keys(parts.flags).flatMap((f) => flagSpec(parts.verb, f)?.requires ?? []), ...verbGate])]; + const settings: Record = Object.fromEntries(Object.entries(parts.flags).map(([f, v]) => [f.replace(/^--/, '').replaceAll('-', '_'), v])); + if ('load' in settings) settings.load_sources = true; + if ('no_load' in settings) settings.load_sources = false; + delete settings.load; + delete settings.no_load; + if (typeof settings.vars === 'string' && /^\{\s*\}$/.test(settings.vars)) delete settings.vars; + if (parts.verb === 'load') Object.assign(settings, settings.reload ? {} : { load_sources: true }, { run_tests: false, run_audits: false }); + // As on the server: run.vars for variables, and run.load over the managed sources an explicit load writes. + const loads = settings.load_sources === true || settings.reload === true ? writeSet.filter(isManaged) : []; + const flagGates = Object.keys(parts.flags).flatMap((f) => flagSpec(parts.verb, f)?.requires ?? []).filter((a) => a !== 'run.vars' && a !== 'run.load'); + const gates = [...new Set([...flagGates, ...('vars' in settings ? ['run.vars' as const] : []), ...(loads.length ? ['run.load' as const] : [])])]; const permissions = [ decide('run.adhoc', covered), ...gates.map((action) => decide(action, this.#ctx.can(action))), @@ -1140,7 +1150,7 @@ export class MockOperate { verb: parts.verb, selection, writeSet, - settings: Object.fromEntries(Object.entries(parts.flags).map(([f, v]) => [f.replace(/^--/, '').replaceAll('-', '_'), v])), + settings, permissions, error: null, launchable: permissions.every((p) => p.allowed) diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2536a86525..a8a1eed47e 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -501,7 +501,7 @@ for upstream or downstream nodes. Positional selectors work as on the command li | `--full-refresh`, `--start-cursor`, `--end-cursor`, `--start-cursor-ts`, `--end-cursor-ts`, `--start-cursor-int`, `--end-cursor-int` | `build` | `run.rewrite_history` | | `--allow-snapshot-full-refresh`, `--allow-snapshot-schema-change`, `--allow-table-type-downgrade`, `--allow-retention-decrease` | `build` | `run.rewrite_history` | | `--warehouse` | All | `run.warehouse`, and the warehouse configured for the target on the location | -| `--vars` | All | `run.vars` | +| `--vars` | All | `run.vars`, also to preview a plan with them | | `--load`, `--reload`, and the `load` verb | `build`, `load` | `run.load` on the sources the run loads | | `--max-microbatches`, `--no-load`, `--no-tests`, `--no-audits`, `--no-python`, `--fail-fast`, `--selection-diagnostics`, `--verbose` | `build` | `run.adhoc` | | `--concurrency` | `build`, `test`, `seed`, `audit` | `run.adhoc` | From ed534715656a48f044cea87b2a26719f3257fc81 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:55:28 +0000 Subject: [PATCH 5/5] fix(ui): the mock's build write set includes the managed sources it loads --- ui/src/lib/api/mock/operate-store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index 345f5f8c83..d7a4c7be77 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1120,7 +1120,7 @@ export class MockOperate { const writable = selection.filter((n) => !isSource(n)); const writeSet = parts.verb === 'build' - ? writable.filter((n) => !(parts.flags['--no-load'] && snapshotNode(n)?.kind === 'loader')) + ? [...new Set([...writable, ...selection.flatMap((n) => snapshotNode(n)?.upstream ?? []).filter(isManaged)])].filter((n) => !(parts.flags['--no-load'] && isManaged(n))) : parts.verb === 'seed' ? writable.filter((n) => snapshotNode(n)?.kind === 'seed') : parts.verb === 'load'