Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1410,7 +1410,7 @@ export interface components {
* @description Closed permission vocabulary; unknown actions fail deserialization.
* @enum {string}
*/
Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view";
Action: "view" | "view.sql" | "view.evidence" | "job.run" | "run.cancel" | "run.rerun" | "run.rewrite_history" | "run.prioritise" | "run.warehouse" | "run.vars" | "run.load" | "trigger.toggle" | "sensor.reset_cursor" | "run.adhoc" | "connection.test" | "user.manage" | "role.assign" | "audit.view";
/** @description What launching the command would run and whether this person may launch it. */
AdhocCheck: {
/** @description Why the command cannot run regardless of permissions, such as a refused flag or an unknown selector. */
Expand Down Expand Up @@ -2785,6 +2785,12 @@ export interface components {
kind: "check_finished";
node_key: string;
outcome: components["schemas"]["NodeOutcome"];
} | {
/** @enum {string} */
kind: "compiled";
node_hashes: {
[key: string]: string;
};
} | {
/** @enum {string} */
kind: "unresponsive";
Expand Down
68 changes: 55 additions & 13 deletions crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs};

use crate::adhoc::constants::{
ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST,
FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST,
POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND,
UNIT_TEST_VERBS,
FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_BUILD_SETTINGS,
LOAD_SOURCES_DEST, LOAD_VERB, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, RELOAD_DEST,
SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, VARS_CONVERTER, VARS_DEST,
};
use crate::argv::main::command_settings_arguments::command_settings_arguments;
use crate::argv::main::job_settings_arguments::job_settings_arguments;
Expand All @@ -44,7 +44,7 @@ impl CommandPlanner for CliPlanner {
.ok_or_else(|| "enter a command, such as build --select orders".to_owned())?;
if !ADHOC_VERBS.contains(&verb.as_str()) {
return Err(format!(
"sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit or freshness"
"sqb {verb} is not available as an ad-hoc run; use build, test, seed, audit, freshness or load"
));
}
let parsed = parse(command)?;
Expand All @@ -66,9 +66,21 @@ impl CommandPlanner for CliPlanner {
if dest == CHANGED_DEST && settings.contains_key(CHANGED_DEST) {
continue;
}
settings.insert(dest.clone(), setting(dest, value)?);
let setting = setting(dest, value)?;
if dest == VARS_DEST && setting.as_object().is_some_and(serde_json::Map::is_empty) {
continue;
}
settings.insert(dest.clone(), setting);
}
if verb == LOAD_VERB {
for (dest, value) in LOAD_BUILD_SETTINGS {
settings.insert(dest.to_owned(), json!(value));
}
if settings.get(RELOAD_DEST) == Some(&Json::Bool(true)) {
settings.remove(LOAD_SOURCES_DEST);
}
}
if verb == BUILD_VERB {
if verb == BUILD_VERB || verb == LOAD_VERB {
job_settings_arguments(&settings)?;
} else {
command_settings_arguments(verb, &settings)?;
Expand All @@ -88,6 +100,7 @@ impl CommandPlanner for CliPlanner {
seeds
}
FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?,
LOAD_VERB => managed_sources(publication, &select, &exclude)?,
_ => {
let python = verb == BUILD_VERB
&& settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false));
Expand All @@ -102,11 +115,22 @@ impl CommandPlanner for CliPlanner {
}
}

/// Whole numbers convert as the parser needs them; paths and variables stay text so the refusal can name them.
/// Whole numbers convert as the parser needs them; `--vars` must be a JSON object; paths stay text so the refusal can name them.
struct PlannerHost;

impl ArgvHost for PlannerHost {
fn convert(&mut self, converter: &str, text: &str) -> Result<Value, TypeFailure> {
if converter == VARS_CONVERTER {
return match serde_json::from_str::<Json>(text) {
Ok(Json::Object(_)) => Ok(Value::Str(text.to_owned())),
Ok(_) => Err(TypeFailure::ArgumentType(
"--vars must be a JSON object".to_owned(),
)),
Err(error) => Err(TypeFailure::ArgumentType(format!(
"--vars must be valid JSON: {error}"
))),
};
}
if !INTEGER_CONVERTERS.contains(&converter) {
return Ok(Value::Str(text.to_owned()));
}
Expand Down Expand Up @@ -165,12 +189,6 @@ fn setting(dest: &str, value: &Value) -> Result<Json, String> {
if let Some((_, flag, reason)) = REFUSED_DESTS.iter().find(|(name, _, _)| *name == dest) {
return Err(format!("{flag} is not available for ad-hoc runs: {reason}"));
}
if dest == LOAD_SOURCES_DEST && *value == Value::Bool(true) {
return Err(
"--load is not available for ad-hoc runs: source loads are not part of ad-hoc runs yet"
.into(),
);
}
if !ALLOWED_DESTS.contains(&dest) {
return Err(format!(
"{} is not available for ad-hoc runs",
Expand All @@ -183,6 +201,8 @@ fn setting(dest: &str, value: &Value) -> Result<Json, String> {
.parse::<i64>()
.map(|number| json!(number))
.map_err(|_| format!("{} must be a whole number", flag_of(dest))),
Value::Str(text) if dest == VARS_DEST => serde_json::from_str(text)
.map_err(|error| format!("--vars must be valid JSON: {error}")),
Value::Str(text) => Ok(json!(text)),
_ => Err(format!(
"{} is not available for ad-hoc runs",
Expand Down Expand Up @@ -362,6 +382,28 @@ fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap<String
selections
}

/// Managed sources `sqb load` runs; a source only a loader writes is not published as a resource but is managed.
fn managed_sources(
publication: (&[PlannerResource], &PublishedJobs),
select: &[String],
exclude: &[String],
) -> Result<BTreeSet<String>, String> {
let unmanaged: BTreeSet<&str> = publication
.0
.iter()
.filter(|resource| resource.loader.is_none())
.map(|resource| resource.key.as_str())
.collect();
let sources: BTreeSet<String> = resolved(publication, select, exclude, true)?
.into_iter()
.filter(|key| key.starts_with("source:") && !unmanaged.contains(key.as_str()))
.collect();
if sources.is_empty() {
return Err("the selectors match no managed sources in the current publication".into());
}
Ok(sources)
}

/// Sources `sqb freshness` observes: selected or upstream sources, less those the exclusions reach.
fn freshness_sources(
publication: (&[PlannerResource], &PublishedJobs),
Expand Down
39 changes: 27 additions & 12 deletions crates/sqlbuild-cli/src/adhoc/constants.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
//! Which commands and flags an ad-hoc run accepts; anything not listed is refused.

/// Verbs a person may plan; workers launch only those the control plane lists.
pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "freshness"];
pub(crate) const ADHOC_VERBS: [&str; 6] = ["build", "test", "seed", "audit", "freshness", "load"];

/// The verb that runs only the loaders of managed sources, planned as a build of those sources.
pub(crate) const LOAD_VERB: &str = "load";

/// The build settings a load runs with: loaders on, no tests or audits, as `sqb load` does.
pub(crate) const LOAD_BUILD_SETTINGS: [(&str, bool); 3] = [
(LOAD_SOURCES_DEST, true),
("run_tests", false),
("run_audits", false),
];

/// The destination `--load` and `--no-load` share.
pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources";

/// The destination `--reload` sets; it implies loading.
pub(crate) const RELOAD_DEST: &str = "reload";

/// The destination `--vars` sets; its JSON object becomes the run's `vars` setting.
pub(crate) const VARS_DEST: &str = "vars";

/// The converter that reads `--vars` as a JSON object.
pub(crate) const VARS_CONVERTER: &str = "parse_cli_vars";

/// The verb whose settings must also form valid job settings.
pub(crate) const BUILD_VERB: &str = "build";
Expand Down Expand Up @@ -30,14 +52,11 @@ pub(crate) const CHANGED_DEST: &str = "changed";
/// The destination `--no-python` clears; only a build with Python nodes resolves them.
pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python";

/// The destination `--load` and `--no-load` share; only `--no-load` is accepted.
pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources";

/// Parser destinations that carry selectors, resolved to exact node keys on the server.
pub(crate) const SELECTOR_DESTS: [&str; 3] = ["select", "exclude", "positional_select"];

/// Destinations an ad-hoc run may set; the control plane gates history rewrites and warehouses.
pub(crate) const ALLOWED_DESTS: [&str; 29] = [
pub(crate) const ALLOWED_DESTS: [&str; 31] = [
"max_microbatches",
"changed",
"reach",
Expand Down Expand Up @@ -67,6 +86,8 @@ pub(crate) const ALLOWED_DESTS: [&str; 29] = [
"allow_retention_decrease",
"warehouse",
"load_sources",
"reload",
"vars",
];

/// Parser converters that take whole numbers; the table names them as Python does.
Expand All @@ -76,23 +97,17 @@ pub(crate) const INTEGER_CONVERTERS: [&str; 2] = ["int", POSITIVE_INTEGER_CONVER
pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer";

/// Refused destinations: the flag people typed and why an ad-hoc run cannot honour it.
pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 25] = [
pub(crate) const REFUSED_DESTS: [(&str, &str, &str); 23] = [
(
"select_file",
"--select-file",
"it reads a file on the worker",
),
(
"reload",
"--reload",
"source loads are not part of ad-hoc runs yet",
),
(
"allow_missing_migration_origin",
"--allow-missing-migration-origin",
"it can lose data",
),
("vars", "--vars", "variables change the published SQL"),
("defer_to", "--defer-to", "it reads another target"),
(
"defer_sources_to",
Expand Down
69 changes: 61 additions & 8 deletions crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -162,19 +162,48 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
expected_plan: Ok((&["model:customers"], json!({}))),
},
AdhocPlanTestCase {
description: "variables are refused",
command: &["build", "--vars", "{\"day\": 1}"],
expected_plan: Err("--vars is not available for ad-hoc runs"),
description: "variables become a JSON object setting, as a job's",
command: &[
"build",
"-s",
"customers",
"--vars",
"{\"day\": 1, \"region\": \"eu\"}",
],
expected_plan: Ok((
&["model:customers"],
json!({"vars": {"day": 1, "region": "eu"}}),
)),
},
AdhocPlanTestCase {
description: "empty variables are no variables",
command: &["build", "-s", "customers", "--vars", "{}"],
expected_plan: Ok((&["model:customers"], json!({}))),
},
AdhocPlanTestCase {
description: "variables must be an object",
command: &["build", "--vars", "[1]"],
expected_plan: Err("--vars must be a JSON object"),
},
AdhocPlanTestCase {
description: "variables must be JSON",
command: &["build", "--vars", "{day: 1}"],
expected_plan: Err("--vars must be valid JSON"),
},
AdhocPlanTestCase {
description: "a selector file is refused",
command: &["build", "--select-file", "selection.txt"],
expected_plan: Err("--select-file is not available"),
},
AdhocPlanTestCase {
description: "loading sources is refused",
command: &["build", "--load"],
expected_plan: Err("--load is not available"),
description: "--load becomes a setting for the server to gate",
command: &["build", "-s", "customers", "--load"],
expected_plan: Ok((&["model:customers"], json!({"load_sources": true}))),
},
AdhocPlanTestCase {
description: "--reload becomes a setting for the server to gate",
command: &["build", "-s", "customers", "--reload"],
expected_plan: Ok((&["model:customers"], json!({"reload": true}))),
},
AdhocPlanTestCase {
description: "another target is refused",
Expand Down Expand Up @@ -217,9 +246,9 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
expected_plan: Err("returns"),
},
AdhocPlanTestCase {
description: "loads are not ad-hoc runs yet",
description: "a load needs managed sources",
command: &["load"],
expected_plan: Err("sqb load is not available as an ad-hoc run"),
expected_plan: Err("the selectors match no managed sources"),
},
AdhocPlanTestCase {
description: "commands that are not runs are refused",
Expand Down Expand Up @@ -471,6 +500,30 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co
command: &["build", "--select", "loader:stage_shipments"],
expected_plan: Ok((&["source:stage_shipments"], json!({}))),
},
AdhocPlanTestCase {
description: "a load without selectors runs every managed source as a build without tests or audits",
command: &["load"],
expected_plan: Ok((
&["source:raw_shipments", "source:stage_shipments"],
json!({"load_sources": true, "run_audits": false, "run_tests": false}),
)),
},
AdhocPlanTestCase {
description: "a load selects a source by name and leaves the models that read it",
command: &[
"load",
"--select",
"raw_shipments",
"+shipments",
"--reload",
"--vars",
"{\"day\": 1}",
],
expected_plan: Ok((
&["source:raw_shipments", "source:stage_shipments"],
json!({"reload": true, "run_audits": false, "run_tests": false, "vars": {"day": 1}}),
)),
},
AdhocPlanTestCase {
description: "test selects only the SQL nodes a tag shared with Python nodes matches",
command: &["test", "--select", "tag:orders"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use crate::store::main::query::query;
use crate::store::models::{ControlTransaction, Parameter};
use crate::store::types::Row;

const ACTIONS: [Action; 16] = [
const ACTIONS: [Action; 18] = [
Action::View,
Action::ViewSql,
Action::ViewEvidence,
Expand All @@ -22,6 +22,8 @@ const ACTIONS: [Action; 16] = [
Action::RunRewriteHistory,
Action::RunPrioritise,
Action::RunWarehouse,
Action::RunVars,
Action::RunLoad,
Action::TriggerToggle,
Action::SensorResetCursor,
Action::RunAdhoc,
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlbuild-orchestration/src/credentials/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,10 @@ pub enum Action {
RunPrioritise,
#[serde(rename = "run.warehouse")]
RunWarehouse,
#[serde(rename = "run.vars")]
RunVars,
#[serde(rename = "run.load")]
RunLoad,
#[serde(rename = "trigger.toggle")]
TriggerToggle,
#[serde(rename = "sensor.reset_cursor")]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ pub(crate) fn validate_authority(
| RunChange::NodeFinished { .. }
| RunChange::CheckStarted { .. }
| RunChange::CheckFinished { .. }
| RunChange::Compiled { .. }
),
RunAuthority::Worker { run_id, claim_id } => {
run_id == request.run_id
Expand All @@ -81,6 +82,7 @@ pub(crate) fn validate_authority(
| RunChange::NodeFinished { .. }
| RunChange::CheckStarted { .. }
| RunChange::CheckFinished { .. }
| RunChange::Compiled { .. }
| RunChange::Recovered
| RunChange::Finished {
outcome: RunOutcome::Success
Expand Down Expand Up @@ -130,7 +132,8 @@ pub(crate) fn validate_change(
RunChange::NodeStarted { .. }
| RunChange::NodeFinished { .. }
| RunChange::CheckStarted { .. }
| RunChange::CheckFinished { .. } => {
| RunChange::CheckFinished { .. }
| RunChange::Compiled { .. } => {
matches!(state.status, Running | Unresponsive)
}
RunChange::Unresponsive => state.status == Running,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ pub(crate) async fn project_run_state(
match &request.change {
RunChange::LaunchFailed { .. }
| RunChange::CheckStarted { .. }
| RunChange::CheckFinished { .. } => {}
| RunChange::CheckFinished { .. }
| RunChange::Compiled { .. } => {}
RunChange::NodeStarted { .. } | RunChange::NodeFinished { .. } => {
return project_node(transaction, request, &state.node_counts_json).await;
}
Expand Down
Loading
Loading