Skip to content

Commit b296cb4

Browse files
committed
test(server): table-driven recent executions and preview helpers
Move the location plan-from-copy helper out of the test module, drive the --changed preview and recent executions tests from test cases, keep the synced copy's consumption read in its own layer, and list the consumption state reader as a synced-copy preview branch.
1 parent d8c8ff0 commit b296cb4

11 files changed

Lines changed: 197 additions & 149 deletions

File tree

‎crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,7 @@ pub struct WarehouseSyncDue {
435435
pub struct PreviewCopyRead {
436436
pub session_id: Uuid,
437437
pub generation: i64,
438-
/// Whether the copy carries consumption history, which only `--changed` plans read; sent only
439-
/// when set, so other plans still reach servers that predate it.
438+
/// Whether the copy carries consumption history; sent only when set, as older servers reject it.
440439
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
441440
pub consumption: bool,
442441
}
@@ -470,8 +469,7 @@ pub struct PreviewCopy {
470469
pub fingerprints: Vec<WarehouseFingerprintRecord>,
471470
pub latest_runs: Vec<PreviewLatestRun>,
472471
pub observations: Vec<WarehouseObservationRecord>,
473-
/// Every mirrored consumption row when the read asked for them; locations that never ask
474-
/// never receive the field.
472+
/// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it.
475473
#[serde(default, skip_serializing_if = "Vec::is_empty")]
476474
pub consumption: Vec<WarehouseConsumptionRecord>,
477475
}

‎crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,9 @@ impl ConsumptionConnection {
456456
) -> PyResult<Vec<ConsumptionRow>> {
457457
if let Ok(copy) = self.adapter.bind(py).downcast::<SyncedCopyAdapter>() {
458458
return Ok(recent_executions(
459-
&copy.get().consumption(location),
459+
&copy
460+
.get()
461+
.consumption((location.database.as_deref(), &location.schema)),
460462
RECENT_EXECUTIONS,
461463
));
462464
}
@@ -470,7 +472,9 @@ impl ConsumptionConnection {
470472
execution_ids: &[&str],
471473
) -> PyResult<Vec<ConsumptionRow>> {
472474
if let Ok(copy) = self.adapter.bind(py).downcast::<SyncedCopyAdapter>() {
473-
let mut rows: Vec<ConsumptionRow> = copy.get().consumption(location);
475+
let mut rows: Vec<ConsumptionRow> = copy
476+
.get()
477+
.consumption((location.database.as_deref(), &location.schema));
474478
rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str()));
475479
return Ok(rows);
476480
}

‎crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use sqlbuild_state::execution_history::constants::MEMORY_PATH;
2424
use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND};
2525

2626
use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with};
27-
use crate::bindings::_helpers::native_host::consumption_state::Location;
2827
use crate::bindings::_helpers::native_host::host;
2928
use crate::bindings::_helpers::native_host::sqlite_connection::{SqliteConnection, SqliteCursor};
3029
use crate::bindings::_helpers::records::fields::field as native_field;
@@ -254,17 +253,17 @@ impl SyncedCopyAdapter {
254253
None
255254
}
256255

257-
/// Every copied consumption row of the table in `location`.
258-
pub(crate) fn consumption(&self, location: &Location) -> Vec<ConsumptionRow> {
256+
/// Every copied consumption row of the table in `(database, schema)`.
257+
pub(crate) fn consumption(
258+
&self,
259+
(database, schema): (Option<&str>, &str),
260+
) -> Vec<ConsumptionRow> {
259261
let mut rows: Vec<ConsumptionRow> = Vec::new();
260262
for record in &self.copy.consumption {
261-
if !same(Some(&record.state_schema), Some(&location.schema))
262-
|| !(location.database.is_none()
263+
if !same(Some(&record.state_schema), Some(schema))
264+
|| !(database.is_none()
263265
|| record.state_database.is_none()
264-
|| same(
265-
record.state_database.as_deref(),
266-
location.database.as_deref(),
267-
))
266+
|| same(record.state_database.as_deref(), database))
268267
{
269268
continue;
270269
}

‎crates/sqlbuild-state/src/consumption/main/recent_executions.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ use std::collections::{BTreeMap, BTreeSet};
44

55
use crate::consumption::models::{ConsumptionRow, RelationKey};
66

7-
/// Rows of the newest `limit` executions per relation, timing each execution by its earliest row,
8-
/// as `recent_executions_sql` selects them in the warehouse.
7+
/// Rows of the newest `limit` executions per relation by earliest row, as `recent_executions_sql`.
98
pub fn recent_executions(rows: &[ConsumptionRow], limit: usize) -> Vec<ConsumptionRow> {
109
let mut started: BTreeMap<(&RelationKey, &str), i64> = BTreeMap::new();
1110
for row in rows {

‎crates/sqlbuild-state/src/consumption/tests/helpers.rs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,3 +761,27 @@ pub(crate) fn two_jobs() -> Scenario {
761761

762762
pub(crate) const JOB_A: (&str, &[&str]) = ("job_a", &["stg_orders", "int_orders"]);
763763
pub(crate) const JOB_B: (&str, &[&str]) = ("job_b", &["fct_revenue", "rpt_pnl"]);
764+
765+
/// One succeeded row of `execution`, writing the `main` relation named `relation`.
766+
pub(crate) fn recorded_row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow {
767+
ConsumptionRow {
768+
row: ConsumptionAppend {
769+
execution_id: execution.into(),
770+
phase: "succeeded".into(),
771+
run_id: format!("run-{execution}"),
772+
node: NodeKey {
773+
kind: "model".into(),
774+
name: relation.into(),
775+
},
776+
relation: RelationKey {
777+
database: None,
778+
schema: "main".into(),
779+
name: relation.into(),
780+
},
781+
version_hash: "version".into(),
782+
input_count: 0,
783+
input: None,
784+
},
785+
recorded_at,
786+
}
787+
}
Lines changed: 51 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,61 @@
11
use crate::consumption::main::recent_executions::recent_executions;
2-
use crate::consumption::models::{ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey};
3-
4-
fn row(relation: &str, execution: &str, recorded_at: i64) -> ConsumptionRow {
5-
ConsumptionRow {
6-
row: ConsumptionAppend {
7-
execution_id: execution.into(),
8-
phase: "succeeded".into(),
9-
run_id: format!("run-{execution}"),
10-
node: NodeKey {
11-
kind: "model".into(),
12-
name: relation.into(),
13-
},
14-
relation: RelationKey {
15-
database: None,
16-
schema: "main".into(),
17-
name: relation.into(),
18-
},
19-
version_hash: "version".into(),
20-
input_count: 0,
21-
input: None,
22-
},
23-
recorded_at,
24-
}
25-
}
2+
use crate::consumption::tests::helpers::recorded_row;
3+
use crate::consumption::tests::test_types::RecentExecutionsTestCase;
264

275
#[test]
286
fn given_more_executions_than_the_limit_when_choosing_recent_then_the_newest_starts_win_per_relation()
297
{
30-
let mut rows: Vec<ConsumptionRow> = Vec::new();
31-
for execution in 1..=4 {
32-
// `orders-1` started first but wrote its last row after every other execution.
33-
let finished: i64 = if execution == 1 {
34-
50
35-
} else {
36-
execution * 10 + 5
37-
};
38-
rows.push(row(
8+
// `orders-1` started first but wrote its last row after every other execution.
9+
let mut late_finisher = Vec::new();
10+
for (execution, started, finished) in [(1, 10, 50), (2, 20, 25), (3, 30, 35), (4, 40, 45)] {
11+
late_finisher.push(recorded_row(
12+
"orders",
13+
&format!("orders-{execution}"),
14+
started,
15+
));
16+
late_finisher.push(recorded_row(
3917
"orders",
4018
&format!("orders-{execution}"),
41-
execution * 10,
19+
finished,
4220
));
43-
rows.push(row("orders", &format!("orders-{execution}"), finished));
4421
}
45-
rows.push(row("customers", "customers-1", 1));
46-
47-
let mut kept: Vec<String> = recent_executions(&rows, 2)
48-
.into_iter()
49-
.map(|row| row.row.execution_id)
50-
.collect();
51-
kept.dedup();
52-
53-
assert_eq!(
54-
kept,
55-
["orders-3", "orders-4", "customers-1"],
56-
"each relation keeps its newest two executions by earliest row, with every row of each"
57-
);
58-
assert_eq!(recent_executions(&rows, 2).len(), 5);
22+
late_finisher.push(recorded_row("customers", "customers-1", 1));
23+
let test_cases = [
24+
RecentExecutionsTestCase {
25+
description: "an execution that started first but wrote last is the oldest; other relations keep theirs",
26+
rows: late_finisher,
27+
limit: 2,
28+
expected_executions: vec!["orders-3", "orders-4", "customers-1"],
29+
expected_rows: 5,
30+
},
31+
RecentExecutionsTestCase {
32+
description: "equal start times keep the larger execution id, as the warehouse query does",
33+
rows: vec![
34+
recorded_row("orders", "orders-a", 10),
35+
recorded_row("orders", "orders-b", 10),
36+
],
37+
limit: 1,
38+
expected_executions: vec!["orders-b"],
39+
expected_rows: 1,
40+
},
41+
];
42+
for test_case in test_cases {
43+
let kept = recent_executions(&test_case.rows, test_case.limit);
44+
let mut executions: Vec<&str> = kept
45+
.iter()
46+
.map(|row| row.row.execution_id.as_str())
47+
.collect();
48+
executions.dedup();
49+
assert_eq!(
50+
executions, test_case.expected_executions,
51+
"{}",
52+
test_case.description
53+
);
54+
assert_eq!(
55+
kept.len(),
56+
test_case.expected_rows,
57+
"{}",
58+
test_case.description
59+
);
60+
}
5961
}

‎crates/sqlbuild-state/src/consumption/tests/test_types.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,14 @@ pub(crate) struct Scenario {
8787
pub(crate) clock: i64,
8888
}
8989

90+
pub(crate) struct RecentExecutionsTestCase {
91+
pub(crate) description: &'static str,
92+
pub(crate) rows: Vec<ConsumptionRow>,
93+
pub(crate) limit: usize,
94+
pub(crate) expected_executions: Vec<&'static str>,
95+
pub(crate) expected_rows: usize,
96+
}
97+
9098
pub(crate) struct DecideTestCase {
9199
pub(crate) description: &'static str,
92100
pub(crate) outcomes: Vec<(String, crate::consumption::models::ChangedOutcome)>,

‎fensu.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1665,6 +1665,7 @@ allowed_paths = [
16651665
"crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs=A location's requested plan names the synced copy for its sqb plan child.",
16661666
"crates/sqlbuild-core/src/plan_display/constants.rs=Names the synced-copy environment variable.",
16671667
"crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs=Preview branch: a refused freshness observation leaves the source unknown.",
1668+
"crates/sqlbuild-python/src/bindings/_helpers/native_host/consumption_state.rs=Preview branch: --changed reads consumption history from the copy instead of the warehouse table.",
16681669
"crates/sqlbuild-python/src/bindings/_helpers/native_host/registration.rs=Registers the synced copy's refusal error with Python.",
16691670
"crates/sqlbuild-python/src/bindings/_helpers/native_host/synced_copy.rs=Defines the synced-copy adapter.",
16701671
"crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=Only sqb plan activates the synced copy and reports what it could not know.",

‎tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,14 @@ class AdhocRunTestCase:
408408
expected_worker_log: str = ""
409409

410410

411+
@dataclass(frozen=True)
412+
class ChangedPlanPreviewTestCase:
413+
description: str
414+
edited_model: str
415+
edit: tuple[str, str]
416+
expected_changed: dict[str, Any]
417+
418+
411419
@dataclass(frozen=True)
412420
class PlanPreviewTestCase:
413421
description: str

‎tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@
6868
WorkerDeployment,
6969
WorkerExecutionTestCase,
7070
)
71+
from tests.integration.src.sqlbuild.compiler.compiled_project.publication.catalog.helpers import (
72+
request_plan,
73+
)
7174
from tests.integration.src.sqlbuild.compiler.compiled_project.publication.docker.helpers import (
7275
docker_deployment,
7376
)
@@ -1601,3 +1604,54 @@ def create_raw_orders(warehouse: Path) -> None:
16011604
_ = connection.execute(
16021605
"CREATE TABLE main.raw_orders AS SELECT CAST(1 AS BIGINT) AS order_id"
16031606
)
1607+
1608+
1609+
def plan_from_copy(
1610+
tmp_path: Path, project: Path, selection: list[str], settings: dict[str, object]
1611+
) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]:
1612+
"""Sync `project` to a location, lock its warehouse, then plan `selection` from the copy.
1613+
1614+
Returns the refused live plan run while the lock was held and the location's plan document.
1615+
"""
1616+
control: Path = tmp_path / "control/.sqlbuild/control.sqlite3"
1617+
log: Path = tmp_path / "location.log"
1618+
started: datetime = datetime.now(UTC)
1619+
with (
1620+
snapshot_api(tmp_path / "control") as api,
1621+
location_process(
1622+
[
1623+
str(Path(sys.executable).with_name("sqb")),
1624+
"--project-dir",
1625+
str(project),
1626+
"location",
1627+
"serve",
1628+
"--server",
1629+
api.url,
1630+
"--location-id",
1631+
api.location_id,
1632+
"--target",
1633+
"development",
1634+
],
1635+
log,
1636+
tmp_path / "location.stdout",
1637+
env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer},
1638+
) as location,
1639+
):
1640+
_ = wait_for_generation(api, "development", 1, location, log)
1641+
_ = wait_for_warehouse_sync_health(
1642+
api,
1643+
{"failure": None, "tables": ["catalog", "consumption", "fingerprints", "observations"]},
1644+
location,
1645+
log,
1646+
after=started,
1647+
)
1648+
lock: duckdb.DuckDBPyConnection = hold_write_lock(project / "warehouse.db")
1649+
try:
1650+
refused: subprocess.CompletedProcess[str] = live_plan(project, selection)
1651+
request_id: str = request_plan(control, selection, settings)
1652+
result: dict[str, Any] = wait_for_location_work(api, request_id, location, log)
1653+
finally:
1654+
lock.close()
1655+
outcome: dict[str, Any] = result["result_json"]["outcome"]
1656+
assert outcome["kind"] == "plan", outcome
1657+
return refused, outcome["document"]

0 commit comments

Comments
 (0)