Skip to content

Commit b286141

Browse files
authored
feat(server): ad-hoc --vars, --load, --reload and load runs gated by run.vars and run.load (#1484)
* feat(server): plan ad-hoc --vars, --load, --reload and load, gated by run.vars and run.load (WIP) * feat(server): record compiled hashes for ad-hoc variables and gate loads in the UI * fix(ui): describe the --vars, --load and --reload options * fix(server): plan previews with variables need run.vars; align the load mock with the server * fix(ui): the mock's build write set includes the managed sources it loads
1 parent 9339e14 commit b286141

32 files changed

Lines changed: 901 additions & 113 deletions

File tree

‎clients/typescript/src/schema.d.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1410,7 +1410,7 @@ export interface components {
14101410
* @description Closed permission vocabulary; unknown actions fail deserialization.
14111411
* @enum {string}
14121412
*/
1413-
Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view";
1413+
Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "run.vars" | "run.load" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view";
14141414
/** @description What launching the command would run and whether this person may launch it. */
14151415
AdhocCheck: {
14161416
/** @description Why the command cannot run regardless of permissions, such as a refused flag or an unknown selector. */
@@ -2785,6 +2785,12 @@ export interface components {
27852785
kind: "check_finished";
27862786
node_key: string;
27872787
outcome: components["schemas"]["NodeOutcome"];
2788+
} | {
2789+
/** @enum {string} */
2790+
kind: "compiled";
2791+
node_hashes: {
2792+
[key: string]: string;
2793+
};
27882794
} | {
27892795
/** @enum {string} */
27902796
kind: "unresponsive";

‎crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs‎

Lines changed: 55 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs};
2020

2121
use crate::adhoc::constants::{
2222
ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST,
23-
FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST,
24-
POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND,
25-
UNIT_TEST_VERBS,
23+
FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_BUILD_SETTINGS,
24+
LOAD_SOURCES_DEST, LOAD_VERB, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, RELOAD_DEST,
25+
SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, VARS_CONVERTER, VARS_DEST,
2626
};
2727
use crate::argv::main::command_settings_arguments::command_settings_arguments;
2828
use crate::argv::main::job_settings_arguments::job_settings_arguments;
@@ -44,7 +44,7 @@ impl CommandPlanner for CliPlanner {
4444
.ok_or_else(|| "enter a command, such as build --select orders".to_owned())?;
4545
if !ADHOC_VERBS.contains(&verb.as_str()) {
4646
return Err(format!(
47-
"sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit or freshness"
47+
"sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit, freshness or load"
4848
));
4949
}
5050
let parsed = parse(command)?;
@@ -66,9 +66,21 @@ impl CommandPlanner for CliPlanner {
6666
if dest == CHANGED_DEST && settings.contains_key(CHANGED_DEST) {
6767
continue;
6868
}
69-
settings.insert(dest.clone(), setting(dest, value)?);
69+
let setting = setting(dest, value)?;
70+
if dest == VARS_DEST && setting.as_object().is_some_and(serde_json::Map::is_empty) {
71+
continue;
72+
}
73+
settings.insert(dest.clone(), setting);
74+
}
75+
if verb == LOAD_VERB {
76+
for (dest, value) in LOAD_BUILD_SETTINGS {
77+
settings.insert(dest.to_owned(), json!(value));
78+
}
79+
if settings.get(RELOAD_DEST) == Some(&Json::Bool(true)) {
80+
settings.remove(LOAD_SOURCES_DEST);
81+
}
7082
}
71-
if verb == BUILD_VERB {
83+
if verb == BUILD_VERB || verb == LOAD_VERB {
7284
job_settings_arguments(&settings)?;
7385
} else {
7486
command_settings_arguments(verb, &settings)?;
@@ -88,6 +100,7 @@ impl CommandPlanner for CliPlanner {
88100
seeds
89101
}
90102
FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?,
103+
LOAD_VERB => managed_sources(publication, &select, &exclude)?,
91104
_ => {
92105
let python = verb == BUILD_VERB
93106
&& settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false));
@@ -102,11 +115,22 @@ impl CommandPlanner for CliPlanner {
102115
}
103116
}
104117

105-
/// Whole numbers convert as the parser needs them; paths and variables stay text so the refusal can name them.
118+
/// Whole numbers convert as the parser needs them; `--vars` must be a JSON object; paths stay text so the refusal can name them.
106119
struct PlannerHost;
107120

108121
impl ArgvHost for PlannerHost {
109122
fn convert(&mut self, converter: &str, text: &str) -> Result<Value, TypeFailure> {
123+
if converter == VARS_CONVERTER {
124+
return match serde_json::from_str::<Json>(text) {
125+
Ok(Json::Object(_)) => Ok(Value::Str(text.to_owned())),
126+
Ok(_) => Err(TypeFailure::ArgumentType(
127+
"--vars must be a JSON object".to_owned(),
128+
)),
129+
Err(error) => Err(TypeFailure::ArgumentType(format!(
130+
"--vars must be valid JSON: {error}"
131+
))),
132+
};
133+
}
110134
if !INTEGER_CONVERTERS.contains(&converter) {
111135
return Ok(Value::Str(text.to_owned()));
112136
}
@@ -165,12 +189,6 @@ fn setting(dest: &str, value: &Value) -> Result<Json, String> {
165189
if let Some((_, flag, reason)) = REFUSED_DESTS.iter().find(|(name, _, _)| *name == dest) {
166190
return Err(format!("{flag} is not available for ad-hoc runs: {reason}"));
167191
}
168-
if dest == LOAD_SOURCES_DEST && *value == Value::Bool(true) {
169-
return Err(
170-
"--load is not available for ad-hoc runs: source loads are not part of ad-hoc runs yet"
171-
.into(),
172-
);
173-
}
174192
if !ALLOWED_DESTS.contains(&dest) {
175193
return Err(format!(
176194
"{} is not available for ad-hoc runs",
@@ -183,6 +201,8 @@ fn setting(dest: &str, value: &Value) -> Result<Json, String> {
183201
.parse::<i64>()
184202
.map(|number| json!(number))
185203
.map_err(|_| format!("{} must be a whole number", flag_of(dest))),
204+
Value::Str(text) if dest == VARS_DEST => serde_json::from_str(text)
205+
.map_err(|error| format!("--vars must be valid JSON: {error}")),
186206
Value::Str(text) => Ok(json!(text)),
187207
_ => Err(format!(
188208
"{} is not available for ad-hoc runs",
@@ -362,6 +382,28 @@ fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap<String
362382
selections
363383
}
364384

385+
/// Managed sources `sqb load` runs; a source only a loader writes is not published as a resource but is managed.
386+
fn managed_sources(
387+
publication: (&[PlannerResource], &PublishedJobs),
388+
select: &[String],
389+
exclude: &[String],
390+
) -> Result<BTreeSet<String>, String> {
391+
let unmanaged: BTreeSet<&str> = publication
392+
.0
393+
.iter()
394+
.filter(|resource| resource.loader.is_none())
395+
.map(|resource| resource.key.as_str())
396+
.collect();
397+
let sources: BTreeSet<String> = resolved(publication, select, exclude, true)?
398+
.into_iter()
399+
.filter(|key| key.starts_with("source:") && !unmanaged.contains(key.as_str()))
400+
.collect();
401+
if sources.is_empty() {
402+
return Err("the selectors match no managed sources in the current publication".into());
403+
}
404+
Ok(sources)
405+
}
406+
365407
/// Sources `sqb freshness` observes: selected or upstream sources, less those the exclusions reach.
366408
fn freshness_sources(
367409
publication: (&[PlannerResource], &PublishedJobs),

‎crates/sqlbuild-cli/src/adhoc/constants.rs‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,29 @@
11
//! Which commands and flags an ad-hoc run accepts; anything not listed is refused.
22
33
/// Verbs a person may plan; workers launch only those the control plane lists.
4-
pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "freshness"];
4+
pub(crate) const ADHOC_VERBS: [&str; 6] = ["build", "test", "seed", "audit", "freshness", "load"];
5+
6+
/// The verb that runs only the loaders of managed sources, planned as a build of those sources.
7+
pub(crate) const LOAD_VERB: &str = "load";
8+
9+
/// The build settings a load runs with: loaders on, no tests or audits, as `sqb load` does.
10+
pub(crate) const LOAD_BUILD_SETTINGS: [(&str, bool); 3] = [
11+
(LOAD_SOURCES_DEST, true),
12+
("run_tests", false),
13+
("run_audits", false),
14+
];
15+
16+
/// The destination `--load` and `--no-load` share.
17+
pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources";
18+
19+
/// The destination `--reload` sets; it implies loading.
20+
pub(crate) const RELOAD_DEST: &str = "reload";
21+
22+
/// The destination `--vars` sets; its JSON object becomes the run's `vars` setting.
23+
pub(crate) const VARS_DEST: &str = "vars";
24+
25+
/// The converter that reads `--vars` as a JSON object.
26+
pub(crate) const VARS_CONVERTER: &str = "parse_cli_vars";
527

628
/// The verb whose settings must also form valid job settings.
729
pub(crate) const BUILD_VERB: &str = "build";
@@ -30,14 +52,11 @@ pub(crate) const CHANGED_DEST: &str = "changed";
3052
/// The destination `--no-python` clears; only a build with Python nodes resolves them.
3153
pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python";
3254

33-
/// The destination `--load` and `--no-load` share; only `--no-load` is accepted.
34-
pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources";
35-
3655
/// Parser destinations that carry selectors, resolved to exact node keys on the server.
3756
pub(crate) const SELECTOR_DESTS: [&str; 3] = ["select", "exclude", "positional_select"];
3857

3958
/// Destinations an ad-hoc run may set; the control plane gates history rewrites and warehouses.
40-
pub(crate) const ALLOWED_DESTS: [&str; 29] = [
59+
pub(crate) const ALLOWED_DESTS: [&str; 31] = [
4160
"max_microbatches",
4261
"changed",
4362
"reach",
@@ -67,6 +86,8 @@ pub(crate) const ALLOWED_DESTS: [&str; 29] = [
6786
"allow_retention_decrease",
6887
"warehouse",
6988
"load_sources",
89+
"reload",
90+
"vars",
7091
];
7192

7293
/// Parser converters that take whole numbers; the table names them as Python does.
@@ -76,23 +97,17 @@ pub(crate) const INTEGER_CONVERTERS: [&str; 2] = ["int", POSITIVE_INTEGER_CONVER
7697
pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer";
7798

7899
/// Refused destinations: the flag people typed and why an ad-hoc run cannot honour it.
79-
pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 25] = [
100+
pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 23] = [
80101
(
81102
"select_file",
82103
"--select-file",
83104
"it reads a file on the worker",
84105
),
85-
(
86-
"reload",
87-
"--reload",
88-
"source loads are not part of ad-hoc runs yet",
89-
),
90106
(
91107
"allow_missing_migration_origin",
92108
"--allow-missing-migration-origin",
93109
"it can lose data",
94110
),
95-
("vars", "--vars", "variables change the published SQL"),
96111
("defer_to", "--defer-to", "it reads another target"),
97112
(
98113
"defer_sources_to",

‎crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -162,19 +162,48 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
162162
expected_plan: Ok((&["model:customers"], json!({}))),
163163
},
164164
AdhocPlanTestCase {
165-
description: "variables are refused",
166-
command: &["build", "--vars", "{\"day\": 1}"],
167-
expected_plan: Err("--vars is not available for ad-hoc runs"),
165+
description: "variables become a JSON object setting, as a job's",
166+
command: &[
167+
"build",
168+
"-s",
169+
"customers",
170+
"--vars",
171+
"{\"day\": 1, \"region\": \"eu\"}",
172+
],
173+
expected_plan: Ok((
174+
&["model:customers"],
175+
json!({"vars": {"day": 1, "region": "eu"}}),
176+
)),
177+
},
178+
AdhocPlanTestCase {
179+
description: "empty variables are no variables",
180+
command: &["build", "-s", "customers", "--vars", "{}"],
181+
expected_plan: Ok((&["model:customers"], json!({}))),
182+
},
183+
AdhocPlanTestCase {
184+
description: "variables must be an object",
185+
command: &["build", "--vars", "[1]"],
186+
expected_plan: Err("--vars must be a JSON object"),
187+
},
188+
AdhocPlanTestCase {
189+
description: "variables must be JSON",
190+
command: &["build", "--vars", "{day: 1}"],
191+
expected_plan: Err("--vars must be valid JSON"),
168192
},
169193
AdhocPlanTestCase {
170194
description: "a selector file is refused",
171195
command: &["build", "--select-file", "selection.txt"],
172196
expected_plan: Err("--select-file is not available"),
173197
},
174198
AdhocPlanTestCase {
175-
description: "loading sources is refused",
176-
command: &["build", "--load"],
177-
expected_plan: Err("--load is not available"),
199+
description: "--load becomes a setting for the server to gate",
200+
command: &["build", "-s", "customers", "--load"],
201+
expected_plan: Ok((&["model:customers"], json!({"load_sources": true}))),
202+
},
203+
AdhocPlanTestCase {
204+
description: "--reload becomes a setting for the server to gate",
205+
command: &["build", "-s", "customers", "--reload"],
206+
expected_plan: Ok((&["model:customers"], json!({"reload": true}))),
178207
},
179208
AdhocPlanTestCase {
180209
description: "another target is refused",
@@ -217,9 +246,9 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
217246
expected_plan: Err("returns"),
218247
},
219248
AdhocPlanTestCase {
220-
description: "loads are not ad-hoc runs yet",
249+
description: "a load needs managed sources",
221250
command: &["load"],
222-
expected_plan: Err("sqb load is not available as an ad-hoc run"),
251+
expected_plan: Err("the selectors match no managed sources"),
223252
},
224253
AdhocPlanTestCase {
225254
description: "commands that are not runs are refused",
@@ -471,6 +500,30 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co
471500
command: &["build", "--select", "loader:stage_shipments"],
472501
expected_plan: Ok((&["source:stage_shipments"], json!({}))),
473502
},
503+
AdhocPlanTestCase {
504+
description: "a load without selectors runs every managed source as a build without tests or audits",
505+
command: &["load"],
506+
expected_plan: Ok((
507+
&["source:raw_shipments", "source:stage_shipments"],
508+
json!({"load_sources": true, "run_audits": false, "run_tests": false}),
509+
)),
510+
},
511+
AdhocPlanTestCase {
512+
description: "a load selects a source by name and leaves the models that read it",
513+
command: &[
514+
"load",
515+
"--select",
516+
"raw_shipments",
517+
"+shipments",
518+
"--reload",
519+
"--vars",
520+
"{\"day\": 1}",
521+
],
522+
expected_plan: Ok((
523+
&["source:raw_shipments", "source:stage_shipments"],
524+
json!({"reload": true, "run_audits": false, "run_tests": false, "vars": {"day": 1}}),
525+
)),
526+
},
474527
AdhocPlanTestCase {
475528
description: "test selects only the SQL nodes a tag shared with Python nodes matches",
476529
command: &["test", "--select", "tag:orders"],

‎crates/sqlbuild-orchestration/src/authorization/_helpers/person_grants.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use crate::store::main::query::query;
1212
use crate::store::models::{ControlTransaction, Parameter};
1313
use crate::store::types::Row;
1414

15-
const ACTIONS: [Action; 16] = [
15+
const ACTIONS: [Action; 18] = [
1616
Action::View,
1717
Action::ViewSql,
1818
Action::ViewEvidence,
@@ -22,6 +22,8 @@ const ACTIONS: [Action; 16] = [
2222
Action::RunRewriteHistory,
2323
Action::RunPrioritise,
2424
Action::RunWarehouse,
25+
Action::RunVars,
26+
Action::RunLoad,
2527
Action::TriggerToggle,
2628
Action::SensorResetCursor,
2729
Action::RunAdhoc,

‎crates/sqlbuild-orchestration/src/credentials/models.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,10 @@ pub enum Action {
163163
RunPrioritise,
164164
#[serde(rename = "run.warehouse")]
165165
RunWarehouse,
166+
#[serde(rename = "run.vars")]
167+
RunVars,
168+
#[serde(rename = "run.load")]
169+
RunLoad,
166170
#[serde(rename = "trigger.toggle")]
167171
TriggerToggle,
168172
#[serde(rename = "sensor.reset_cursor")]

‎crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/changes.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ pub(crate) fn validate_authority(
7070
| RunChange::NodeFinished { .. }
7171
| RunChange::CheckStarted { .. }
7272
| RunChange::CheckFinished { .. }
73+
| RunChange::Compiled { .. }
7374
),
7475
RunAuthority::Worker { run_id, claim_id } => {
7576
run_id == request.run_id
@@ -81,6 +82,7 @@ pub(crate) fn validate_authority(
8182
| RunChange::NodeFinished { .. }
8283
| RunChange::CheckStarted { .. }
8384
| RunChange::CheckFinished { .. }
85+
| RunChange::Compiled { .. }
8486
| RunChange::Recovered
8587
| RunChange::Finished {
8688
outcome: RunOutcome::Success
@@ -130,7 +132,8 @@ pub(crate) fn validate_change(
130132
RunChange::NodeStarted { .. }
131133
| RunChange::NodeFinished { .. }
132134
| RunChange::CheckStarted { .. }
133-
| RunChange::CheckFinished { .. } => {
135+
| RunChange::CheckFinished { .. }
136+
| RunChange::Compiled { .. } => {
134137
matches!(state.status, Running | Unresponsive)
135138
}
136139
RunChange::Unresponsive => state.status == Running,

‎crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/projection.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ pub(crate) async fn project_run_state(
5858
match &request.change {
5959
RunChange::LaunchFailed { .. }
6060
| RunChange::CheckStarted { .. }
61-
| RunChange::CheckFinished { .. } => {}
61+
| RunChange::CheckFinished { .. }
62+
| RunChange::Compiled { .. } => {}
6263
RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => {
6364
return project_node(transaction, request, &state.node_counts_json).await;
6465
}

0 commit comments

Comments
 (0)