Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
MODEL (description "Order counts per customer, refreshed by the finance job.", materialized table);
SELECT customer_id, count(*) AS revenue FROM __ref("int_orders") GROUP BY customer_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
MODEL (description "Orders with their customer, refreshed by the staging job.", materialized table);
SELECT order_id, customer_id FROM __ref("stg_orders")
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
MODEL (description "Orders exported by a job that nothing triggers.", materialized table);
SELECT order_id FROM __ref("stg_orders")
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
MODEL (description "A report on revenue, refreshed by the finance job.", materialized table);
SELECT sum(revenue) AS total FROM __ref("fct_revenue")
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
MODEL (description "Joins a branch the change reaches now with one that waits for the finance job.", materialized table);
SELECT c.customer_id, count(i.order_id) AS orders, max(f.revenue) AS revenue
FROM __ref("stg_customers") AS c
LEFT JOIN __ref("int_orders") AS i USING (customer_id)
LEFT JOIN __ref("fct_revenue") AS f USING (customer_id)
GROUP BY c.customer_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"""Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger."""

from sqlbuild.orchestration import job, schedule

daily = schedule(cron="0 2 * * *", timezone="UTC")
staging = job(
"staging_daily", description="Stage orders.", select="stg_orders int_orders", schedule=daily
)
finance = job(
"finance_daily",
description="Build revenue reports.",
select="int_orders fct_revenue rpt_pnl",
schedule=daily,
)
exports = job("adhoc_exports", description="Export orders on request.", select="orders_export")
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
MODEL (description "Sales per customer, two levels below changes held by different jobs.", materialized table);
SELECT customer_id, orders FROM __ref("customer_orders") WHERE orders > 0
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily."""

from sqlbuild.orchestration import job, schedule

daily = schedule(cron="0 2 * * *", timezone="UTC")
crm = job(
"crm_daily",
description="Refresh customers.",
select="stg_customers customer_orders",
schedule=daily,
)
sales = job(
"sales_daily",
description="Refresh sales.",
select="stg_orders customer_sales",
schedule=daily,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""A sensor on new source data starts a job that builds only what changed, source data included."""

from sqlbuild.orchestration import job, sensor, source_new_data

orders_refresh = job(
"orders_refresh",
description="Refresh orders when the source has new data.",
select="orders",
settings={"changed": True},
)
on_source_orders = sensor(
"on_source_orders",
description="Refresh orders for each changed source version.",
when=source_new_data(select="source:raw_orders"),
run=orders_refresh,
minimum_interval_seconds=1,
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"description": "A model built before a fingerprint scheme change keeps its old recorded hash; the node is unchanged until its SQL changes.",
"dialect": "duckdb",
"definition": "select order_id,\n amount -- gross\nfrom orders",
"edited_definition": "select order_id, amount * 2 as amount from orders",
"old_scheme_hash": "scheme-orders-hash-before-upgrade",
"metadata": {
"model_name": "scheme_orders",
"config": {"materialized": "table"},
"execution_signature": {},
"local_function_hashes": {}
}
}
10 changes: 1 addition & 9 deletions crates/sqlbuild-analysis/src/sql_tokens/constants.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
//! Fingerprint dialect aliases and the function language fingerprinted like a query.
//! The function language fingerprinted like a query.

/// The fingerprint dialect when a project names none.
pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic";
/// Python's `NATIVE_DIALECT_ALIASES`.
pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [
("postgres", "postgresql"),
("motherduck", "duckdb"),
("sqlserver", "tsql"),
];
/// The function language whose definitions are fingerprinted like queries.
pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql";
Original file line number Diff line number Diff line change
@@ -1,16 +1,10 @@
//! Python's `compute_query_hash`: a query fingerprint under the project's dialect alias.

use crate::sql_tokens::constants::{FINGERPRINT_DIALECT_ALIASES, GENERIC_FINGERPRINT_DIALECT};
use sqlbuild_core::fingerprint_identity::main::fingerprint_dialect::fingerprint_dialect;

use crate::sql_tokens::main::query_fingerprint::query_fingerprint;

/// The fingerprint of `sql` under `dialect` (aliased, or generic when empty or absent).
pub fn compiler_query_fingerprint(sql: &str, dialect: Option<&str>) -> Result<String, String> {
let dialect = match dialect.filter(|name| !name.is_empty()) {
None => GENERIC_FINGERPRINT_DIALECT,
Some(name) => FINGERPRINT_DIALECT_ALIASES
.iter()
.find(|(alias, _)| *alias == name)
.map_or(name, |(_, target)| target),
};
query_fingerprint(sql, dialect)
query_fingerprint(sql, fingerprint_dialect(dialect))
}
12 changes: 10 additions & 2 deletions crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource};
use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs};

use crate::adhoc::constants::{
ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS,
LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS,
ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST,
FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER,
REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS,
};
use crate::argv::main::command_settings_arguments::command_settings_arguments;
use crate::argv::main::job_settings_arguments::job_settings_arguments;
Expand Down Expand Up @@ -51,6 +52,13 @@ impl CommandPlanner for CliPlanner {
selectors.insert(selector, texts(value));
continue;
}
if dest == CHANGED_COUNTS_DEST {
settings.insert(CHANGED_DEST.to_owned(), json!(texts(value)));
continue;
}
if dest == CHANGED_DEST && settings.contains_key(CHANGED_DEST) {
continue;
}
settings.insert(dest.clone(), setting(dest, value)?);
}
if verb == BUILD_VERB {
Expand Down
10 changes: 9 additions & 1 deletion crates/sqlbuild-cli/src/adhoc/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,23 @@ pub(crate) const FRESHNESS_VERB: &str = "freshness";
/// The parser destination holding the verb itself, which is not a setting.
pub(crate) const COMMAND_DEST: &str = "command";

/// The destination holding the kinds `--changed=` counts; the job setting carries them as `changed`.
pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts";

/// The job setting `--changed` becomes.
pub(crate) const CHANGED_DEST: &str = "changed";

/// The destination `--load` and `--no-load` share; only `--no-load` is accepted.
pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources";

/// Parser destinations that carry selectors, resolved to exact node keys on the server.
pub(crate) const SELECTOR_DESTS: [&str; 3] = ["select", "exclude", "positional_select"];

/// Destinations an ad-hoc run may set; the control plane gates history rewrites and warehouses.
pub(crate) const ALLOWED_DESTS: [&str; 27] = [
pub(crate) const ALLOWED_DESTS: [&str; 29] = [
"max_microbatches",
"changed",
"reach",
"run_tests",
"run_audits",
"include_python",
Expand Down
25 changes: 20 additions & 5 deletions crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,26 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
json!({"fail_fast": true}),
)),
},
AdhocPlanTestCase {
description: "--changed with its counts and reach is the job setting a schedule would use",
command: &[
"build",
"-s",
"orders",
"--changed=source_data,definition",
"--reach",
"all",
],
expected_plan: Ok((
&["model:orders"],
json!({"changed": ["definition", "source_data"], "reach": "all"}),
)),
},
AdhocPlanTestCase {
description: "a bare --changed uses the project's [changes] defaults",
command: &["build", "-s", "orders", "--changed"],
expected_plan: Ok((&["model:orders"], json!({"changed": true}))),
},
AdhocPlanTestCase {
description: "an abbreviated flag is the same flag, so it is still detected",
command: &["build", "-s", "orders", "--full-ref"],
Expand Down Expand Up @@ -144,11 +164,6 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select
command: &["build", "--vars", "{\"day\": 1}"],
expected_plan: Err("--vars is not available for ad-hoc runs"),
},
AdhocPlanTestCase {
description: "a flag outside the allowlist is refused by the name people typed",
command: &["build", "--changed"],
expected_plan: Err("--changed is not available for ad-hoc runs"),
},
AdhocPlanTestCase {
description: "a selector file is refused",
command: &["build", "--select-file", "selection.txt"],
Expand Down
12 changes: 12 additions & 0 deletions crates/sqlbuild-cli/src/argv/_helpers/formatting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ fn format_args(action: &ActionSpec, default: &str) -> String {
match action.nargs {
Nargs::Single => name,
Nargs::Optional => format!("[{name}]"),
Nargs::Attached => format!("[={name}]"),
Nargs::ZeroOrMore => format!("[{name} ...]"),
Nargs::OneOrMore => format!("{name} [{name} ...]"),
Nargs::Parser => format!("{name} ..."),
Expand All @@ -252,6 +253,12 @@ fn invocation(action: &ActionSpec) -> String {
if action.nargs == Nargs::Zero {
return options;
}
if action.nargs == Nargs::Attached {
return format!(
"{options}{}",
format_args(action, &dest(action).to_uppercase())
);
}
format!(
"{options} {}",
format_args(action, &dest(action).to_uppercase())
Expand Down Expand Up @@ -438,6 +445,11 @@ fn usage_part(action: &ActionSpec, grouped: bool) -> Option<String> {
let option: &str = &action.option_strings[0];
let part: String = if action.nargs == Nargs::Zero {
option.to_owned()
} else if action.nargs == Nargs::Attached {
format!(
"{option}{}",
format_args(action, &dest(action).to_uppercase())
)
} else {
format!(
"{option} {}",
Expand Down
24 changes: 23 additions & 1 deletion crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use serde_json::Value as Json;

use crate::argv::_helpers::table::root_parser;
use crate::argv::constants::CONCURRENCY_DEST;
use crate::argv::models::{ActionKind, ActionSpec, Default, ParserSpec, Value};
use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, Value};

pub(crate) fn setting_argument(
parser: &ParserSpec,
Expand Down Expand Up @@ -45,6 +45,9 @@ pub(crate) fn setting_argument(
return boolean_argument(name, value, &actions);
}
let option = &action.option_strings[0];
if action.nargs == Nargs::Attached {
return attached_argument(name, value, option);
}
let text = setting_text(name, value, action)?;
if action
.choices
Expand Down Expand Up @@ -83,6 +86,25 @@ fn boolean_argument(
Ok(None)
}

/// `true` as the bare option, `false` as nothing, and a list of strings as `--option=a,b`.
fn attached_argument(name: &str, value: &Json, option: &str) -> Result<Option<String>, String> {
match value {
Json::Bool(true) => Ok(Some(option.to_owned())),
Json::Bool(false) => Ok(None),
Json::Array(items) if !items.is_empty() => {
let texts: Vec<&str> = items
.iter()
.map(Json::as_str)
.collect::<Option<_>>()
.ok_or_else(|| format!("job setting {name} must be true or a list of strings"))?;
Ok(Some(format!("{option}={}", texts.join(","))))
}
_ => Err(format!(
"job setting {name} must be true or a list of strings"
)),
}
}

fn setting_text(name: &str, value: &Json, action: &ActionSpec) -> Result<String, String> {
match action.converter.as_deref() {
Some("int" | "_positive_integer") => {
Expand Down
40 changes: 35 additions & 5 deletions crates/sqlbuild-cli/src/argv/_helpers/normalization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

use crate::argv::_helpers::parsing::{Exit, KnownArgs, Run};
use crate::argv::constants::{
COMMAND_DEST, CONCURRENCY_COMMANDS, CONCURRENCY_ENV_VAR, CURSOR_COMMANDS, DBT_COMMAND,
DBT_DEBUG_COMMAND, DBT_PASSTHROUGH_COMMANDS, DBT_PASSTHROUGH_SEPARATOR, DBT_VERBOSE_OPTION,
DBT_VERBOSE_OPTIONS, DEBUG_OPTION, INT_CONVERTER, JANITOR_COMMAND, NO_COLOR_OPTION,
POSITIONAL_SELECT_COMMANDS, RENAME_COMMAND, SCOPE_COMMAND, SCOPE_DEFAULT_PAGE_SIZE,
SCOPE_GLOBAL_SUMMARY, SCOPE_KIND_DEST, TEST_COMMAND,
CHANGE_COUNTS, COMMAND_DEST, CONCURRENCY_COMMANDS, CONCURRENCY_ENV_VAR, CURSOR_COMMANDS,
DBT_COMMAND, DBT_DEBUG_COMMAND, DBT_PASSTHROUGH_COMMANDS, DBT_PASSTHROUGH_SEPARATOR,
DBT_VERBOSE_OPTION, DBT_VERBOSE_OPTIONS, DEBUG_OPTION, INT_CONVERTER, JANITOR_COMMAND,
NO_COLOR_OPTION, POSITIONAL_SELECT_COMMANDS, RENAME_COMMAND, SCOPE_COMMAND,
SCOPE_DEFAULT_PAGE_SIZE, SCOPE_GLOBAL_SUMMARY, SCOPE_KIND_DEST, TEST_COMMAND,
};
use crate::argv::models::{Namespace, ParserSpec, Value};

Expand Down Expand Up @@ -102,6 +102,8 @@ impl Run<'_> {
"test --inspect cannot be combined with --json or --json-output",
));
}
namespace =
with_changed_counts(namespace).map_err(|message| self.error(parser, &message))?;
if let Some(message) = since_error(&namespace) {
return Err(self.error(parser, message));
}
Expand Down Expand Up @@ -298,3 +300,31 @@ fn subcommand_parser<'a>(parser: &'a ParserSpec, name: &str) -> Option<&'a Parse
.find(|(command, _)| command == name)
.map(|(_, spec)| spec)
}

/// `namespace` with `--changed=COUNTS` split into `changed_counts`; refuses `--reach` alone.
fn with_changed_counts(mut namespace: Namespace) -> Result<Namespace, String> {
if let Value::Str(text) = namespace.value("changed") {
let counts: Vec<&str> = text.split(',').map(str::trim).collect();
if counts.iter().any(|count| !CHANGE_COUNTS.contains(count)) {
return Err(format!(
"--changed lists what counts as a change: one or more of {}, separated by commas, \
not {text:?}",
CHANGE_COUNTS.join(", ")
));
}
let mut listed: Vec<Value> = Vec::new();
for count in CHANGE_COUNTS {
if counts.contains(&count) {
listed.push(Value::Str(count.to_owned()));
}
}
namespace.set("changed", Value::Bool(true));
namespace.set("changed_counts", Value::List(listed));
} else if truthy(&namespace.value("changed")) {
namespace.set("changed_counts", Value::None);
}
if namespace.value("reach") != Value::None && !truthy(&namespace.value("changed")) {
return Err("--reach needs --changed".to_owned());
}
Ok(namespace)
}
6 changes: 5 additions & 1 deletion crates/sqlbuild-cli/src/argv/_helpers/parsing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,11 @@ impl<'p, 'r, 'h> Level<'p, 'r, 'h> {
action_tuples.push((index, self.arg_strings[first..stop].to_vec()));
break;
};
let count: usize = match_argument(spec, b"A")?;
let count: usize = if spec.nargs == Nargs::Attached {
1
} else {
match_argument(spec, b"A")?
};
let second: char = option_string.chars().nth(1).unwrap_or_default();
if count == 1 {
stop = start + 1;
Expand Down
3 changes: 2 additions & 1 deletion crates/sqlbuild-cli/src/argv/_helpers/patterns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ fn nargs_pattern(action: &ActionSpec) -> Vec<Element> {
element("A", 1, 1),
element("-AO", 0, UNBOUNDED),
],
(Nargs::Zero, true) => Vec::new(),
(Nargs::Zero | Nargs::Attached, true) => Vec::new(),
(Nargs::Attached, false) => vec![element("-", 0, UNBOUNDED)],
(Nargs::Zero, false) => vec![element("-", 0, UNBOUNDED)],
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-cli/src/argv/_helpers/table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ fn nargs(json: &Json) -> Nargs {
Json::Number(_) => Nargs::Zero,
Json::String(text) => match text.as_str() {
"?" => Nargs::Optional,
"=?" => Nargs::Attached,
"*" => Nargs::ZeroOrMore,
"+" => Nargs::OneOrMore,
_ => Nargs::Parser,
Expand Down
Loading
Loading