Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""The loader-dependencies example from the loaders docs, as written.

raw_accounts is an intermediate loader without a write_strategy that returns rows, so it writes
them to __loader__raw_accounts as a table.
"""

from sqlbuild.executor.load.models import LoaderContext
from sqlbuild.loaders import loader


@loader
def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]:
"""Load customer accounts."""
return [
{"account_id": 1, "account_name": "Pawnee Parks"},
{"account_id": 2, "account_name": "Eagleton"},
]


@loader(depends_on=[raw_accounts])
def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]:
"""Account metrics."""
accounts = ctx.loader(raw_accounts)
rows = ctx.query(f"SELECT account_id FROM {accounts.destination}")
return [{"account_id": row[0], "metric": "active"} for row in rows.fetchall()]
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
sources:
- name: raw_account_metrics
description: Account metrics read from the intermediate raw_accounts loader.
managed: true
write_strategy: table
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Roles over a loader chain whose middle loader no source declares: running the job needs a grant
# on the intermediate loader's node as well as the model and the source it feeds.
[roles.shipments_runner]
description = "Run the shipments build, including its staging loader"

[[roles.shipments_runner.grants]]
nodes = ["orders", "source:raw_shipments", "loader:stage_shipments"]
permissions = ["job.run"]

[roles.orders_runner]
description = "Run the orders build without the staging loader"

[[roles.orders_runner.grants]]
nodes = ["orders", "source:raw_shipments"]
permissions = ["job.run"]
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
"""A source loader that depends on an intermediate loader no source declares."""
"""A source loader that depends on an intermediate loader no source declares.

A job that selects the model must claim the intermediate loader's relation as well as the source.
"""

from sqlbuild.loaders import loader


@loader
@loader(write_strategy="table")
def stage_shipments(ctx):
"""Stage synthetic shipments in the intermediate loader's own relation."""
return [{"order_id": 9}]
Expand Down
10 changes: 10 additions & 0 deletions benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"""Jobs whose selections are Python nodes, which `--select job:<name>` must select as published."""

from sqlbuild.orchestration import job

customer_counts = job(
"customer_counts",
description="Count customers.",
select="task:count_customers asset:customers_extract",
)
order_counts = job("order_counts", description="Count orders.", select="+task:count_orders")
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
"""A task whose Python upstream another job builds, so a job group must order the two jobs."""

from python.orders.partner_export.node import partner_orders_file

from sqlbuild.tasks import task


@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.")
def request_carrier_collection(ctx):
return None
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""A Python check on the orders model that a job's tag selector matches.

A job matching the check and the model it checks publishes and runs the check with the model. A job
matching the check alone fails publication, naming the check and the model it leaves out.
"""

from sqlbuild.checks import check
from sqlbuild.refs import model


@check(depends_on=model("orders"), tags=["order_checks"])
def orders_present(ctx):
"""Orders are present."""
orders = ctx.relation(model("orders"))
count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0]
return ctx.pass_() if count else ctx.fail(message="no orders")
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""An asset one job builds that a task in another job of the same group depends on."""

from sqlbuild.assets import asset
from sqlbuild.refs import model


@asset(depends_on=model("orders"), tags=["orders"], description="Partner order file.")
def partner_orders_file(ctx):
return None
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"""A job that reaches a managed source through `+`, so its run selects the source's loader chain.

`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the
intermediate stage_shipments loader it depends on. The run records each loader as the source it
writes, which is the node its load reports under.
"""

from sqlbuild.orchestration import job, schedule

annual_orders = job(
"annual_orders",
description="Build orders and everything upstream of it, loaders included.",
select="+orders",
schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"),
)
14 changes: 10 additions & 4 deletions crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,23 @@ use crate::graph::errors::SelectorError;
use crate::graph::models::{JobSelection, ProjectGraph};

/// Resolve every job on the graph without jobs attached, as publication does, then attach them.
pub(crate) fn attach(
pub(crate) fn attach(graph: ProjectGraph, catalog: BTreeMap<String, JobSelection>) -> ProjectGraph {
attach_with(graph, catalog, |graph, select, exclude| {
resolve(graph, select, exclude).map(|keys| keys.into_iter().collect())
})
}

/// [`attach`] with each job's selectors resolved by `resolve_job` on the graph without jobs.
pub(crate) fn attach_with(
mut graph: ProjectGraph,
catalog: BTreeMap<String, JobSelection>,
resolve_job: impl Fn(&ProjectGraph, &[String], &[String]) -> Result<Keys, SelectorError>,
) -> ProjectGraph {
graph.jobs = None;
let mut resolved: BTreeMap<String, Result<Keys, SelectorError>> = BTreeMap::new();
for (name, selection) in &catalog {
let keys: Result<Keys, SelectorError> = match selection {
JobSelection::Job { select, exclude } => {
resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect())
}
JobSelection::Job { select, exclude } => resolve_job(&graph, select, exclude),
JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()),
JobSelection::Group { .. } => continue,
};
Expand Down
18 changes: 18 additions & 0 deletions crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
use std::collections::{BTreeMap, BTreeSet};

use crate::assembly::project::types::ObjectKey;
use crate::graph::errors::SelectorError;
use crate::graph::models::{JobSelection, ProjectGraph};

/// [`super::attach_jobs::attach_jobs`] with each job's selectors resolved by `resolve_job`.
pub fn attach_jobs_with(
graph: ProjectGraph,
catalog: BTreeMap<String, JobSelection>,
resolve_job: impl Fn(
&ProjectGraph,
&[String],
&[String],
) -> Result<BTreeSet<ObjectKey>, SelectorError>,
) -> ProjectGraph {
crate::graph::_helpers::jobs::attach_with(graph, catalog, resolve_job)
}
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/graph/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub mod attach_jobs;
pub mod attach_jobs_with;
pub mod build_project_graph;
pub mod build_resources;
pub mod downstream_edges;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use crate::planner_scope::constants::{MODEL, SOURCE};
use crate::planner_scope::models::LoaderGraph;
use std::collections::{BTreeMap, BTreeSet};

/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them.
/// Managed sources each model's execution deps read and each managed source's loader chain, intermediate loaders included, as `expand_loaders` keys them.
pub fn node_loads(
graph: &LoaderGraph,
upstream: &[(ObjectKey, Vec<ObjectKey>)],
Expand All @@ -24,7 +24,6 @@ pub fn node_loads(
graph
.upstream(chain)
.iter()
.filter(|name| !graph.intermediate(name))
.map(|name| graph.key(name))
.collect()
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec
expected_loads: vec![("model:orders", vec!["source:raw_returns"])],
},
NodeLoadsTestCase {
// stage_orders has no declaring source, so a run that loads it still fails in the worker.
description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate",
// stage_orders has no declaring source; the snapshot publishes it as source:stage_orders.
description: "a chain keeps an intermediate loader and the declared loaders behind it",
loaders: vec![
("raw_orders", vec!["stage_orders"]),
("stage_orders", vec!["raw_customers"]),
Expand All @@ -162,7 +162,10 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec
("source:raw_orders", vec![]),
("source:raw_customers", vec![]),
],
expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])],
expected_loads: vec![(
"source:raw_orders",
vec!["source:stage_orders", "source:raw_customers"],
)],
},
];
let key = |text: &str| {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

use std::collections::HashMap;

use crate::assembly::project::types::ObjectKey;
use crate::definition_changes::constants::PYTHON_NODE_KINDS;

use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY};
use crate::python_selection::models::{InventoryIndexes, InventoryNode};

Expand Down Expand Up @@ -83,3 +86,17 @@ pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes {
paths: paths.rows,
}
}

/// A published node's dependency keys split into its Python upstream names and its SQL keys.
pub(crate) fn split_published_deps(deps: &[ObjectKey]) -> (Vec<String>, Vec<ObjectKey>) {
let mut python: Vec<String> = Vec::new();
let mut sql: Vec<ObjectKey> = Vec::new();
for (kind, name) in deps {
if PYTHON_NODE_KINDS.contains(&kind.as_str()) {
python.push(name.clone());
} else {
sql.push((kind.clone(), name.clone()));
}
}
(python, sql)
}
17 changes: 16 additions & 1 deletion crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ use crate::python_selection::constants::{
UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE,
UNMAPPED_KIND_CODE,
};
use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode};
use crate::python_selection::models::{
NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph,
};
use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT;

pub(crate) type Names = BTreeSet<String>;
Expand Down Expand Up @@ -59,6 +61,19 @@ fn keyed(rows: &[(String, Vec<String>)]) -> HashMap<&str, &[String]> {
.collect()
}

impl<'graph> SelectionContext<'graph> {
pub(crate) fn new(
sql: SqlSelectionGraph<'graph>,
python: &'graph PythonSelectionGraph,
) -> Self {
Self {
sql,
nodes: NodeIndex::new(python),
sql_names: sql_names(sql.graph),
}
}
}

impl<'graph> NodeIndex<'graph> {
pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self {
Self {
Expand Down
Loading
Loading