Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2398,6 +2398,8 @@ export interface components {
};
/** @description What the mirror knows as of `synced_at`; `None` when a required table never synced cleanly. */
PreviewCopy: {
/** @description Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it. */
consumption?: components["schemas"]["WarehouseConsumptionRecord"][];
fingerprints: components["schemas"]["WarehouseFingerprintRecord"][];
latest_runs: components["schemas"]["PreviewLatestRun"][];
observations: components["schemas"]["WarehouseObservationRecord"][];
Expand All @@ -2407,6 +2409,8 @@ export interface components {
};
/** @description A location asks for the mirrored copy a plan preview plans from, as its fenced reader. */
PreviewCopyRead: {
/** @description Whether the copy carries consumption history; sent only when set, as older servers reject it. */
consumption?: boolean;
/** Format: int64 */
generation: number;
/** Format: uuid */
Expand Down Expand Up @@ -3258,6 +3262,27 @@ export interface components {
state_database?: string | null;
state_schema: string;
};
/** @description One input version an execution consumed, or the row marking that the execution started. */
WarehouseConsumptionRecord: {
execution_id?: string | null;
/** Format: int64 */
input_count?: number | null;
input_kind?: string | null;
input_name?: string | null;
input_version?: string | null;
node_name?: string | null;
node_type?: string | null;
phase?: string | null;
/** Format: date-time */
recorded_at: string;
run_id?: string | null;
state_database?: string | null;
state_schema: string;
target_database?: string | null;
target_name?: string | null;
target_schema?: string | null;
version_hash?: string | null;
};
/** @description Fingerprints remain history here; latest-row selection belongs to the existing planner reader. */
WarehouseFingerprintRecord: {
definition: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ pub(super) async fn plan_location(
let read = PreviewCopyRead {
session_id,
generation,
consumption: settings
.get("changed")
.is_some_and(|changed| !matches!(changed, Value::Null | Value::Bool(false))),
};
let copy = match client.preview_copy(&options.target, &read).await {
Ok(copy) => copy,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ const LATEST_RUNS_SQL: &str = "SELECT record_json FROM (SELECT record_json, ROW_
WHERE location_id=$1 AND target=$2 AND record_json ->> 'phase'='succeeded' \
AND record_json ->> 'run_id' IS NOT NULL) runs WHERE newest=1 \
ORDER BY record_json ->> 'node_type', record_json ->> 'node_name'";
/// Every consumption row the warehouse still keeps, which complete scans bound.
const CONSUMPTION_SQL: &str = "SELECT record_json FROM sqb_warehouse_consumption \
WHERE location_id=$1 AND target=$2 ORDER BY observed_at, event_id";
const OBSERVATIONS_SQL: &str = "SELECT record_json FROM (SELECT record_json, kind, node, ROW_NUMBER() OVER \
(PARTITION BY state_database, state_schema, kind, node ORDER BY observed_at DESC, event_id DESC) AS newest \
FROM sqb_warehouse_observations WHERE location_id=$1 AND target=$2) observations WHERE newest=1 \
Expand Down Expand Up @@ -147,6 +150,14 @@ pub(crate) async fn preview_latest_runs(
Ok((transaction, runs))
}

/// Every mirrored consumption row: a build reads the newest executions and any older one by ID.
pub(crate) async fn preview_consumption(
transaction: ControlTransaction,
scope: (Uuid, &str),
) -> Result<(ControlTransaction, Vec<WarehouseConsumptionRecord>), ControlError> {
mirrored(transaction, CONSUMPTION_SQL, scope).await
}

/// Each node's newest observation of each kind.
pub(crate) async fn preview_observations(
transaction: ControlTransaction,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use crate::store::main::commit::commit;
use crate::store::models::ControlStore;
use crate::warehouse_metadata::_helpers::ingestion::begin_ingestion;
use crate::warehouse_metadata::_helpers::mirror_pages::{
preview_fingerprints, preview_latest_runs, preview_observations, preview_relations,
preview_synced_at,
preview_consumption, preview_fingerprints, preview_latest_runs, preview_observations,
preview_relations, preview_synced_at,
};
use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead};

Expand All @@ -23,6 +23,11 @@ pub async fn read_preview_copy(
let (transaction, fingerprints) = preview_fingerprints(transaction, scope).await?;
let (transaction, latest_runs) = preview_latest_runs(transaction, scope).await?;
let (transaction, observations) = preview_observations(transaction, scope).await?;
let (transaction, consumption) = if request.consumption {
preview_consumption(transaction, scope).await?
} else {
(transaction, Vec::new())
};
let (transaction, synced_at) = preview_synced_at(transaction, scope).await?;
commit(transaction).await?;
Ok(PreviewCopy {
Expand All @@ -31,5 +36,6 @@ pub async fn read_preview_copy(
fingerprints,
latest_runs,
observations,
consumption,
})
}
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,9 @@ pub struct WarehouseSyncDue {
pub struct PreviewCopyRead {
pub session_id: Uuid,
pub generation: i64,
/// Whether the copy carries consumption history; sent only when set, as older servers reject it.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub consumption: bool,
}

/// The newest successful build of a node, which a `cursor_max` observation must come from.
Expand Down Expand Up @@ -466,6 +469,9 @@ pub struct PreviewCopy {
pub fingerprints: Vec<WarehouseFingerprintRecord>,
pub latest_runs: Vec<PreviewLatestRun>,
pub observations: Vec<WarehouseObservationRecord>,
/// Every mirrored consumption row when asked for; omitted otherwise, as older locations reject it.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub consumption: Vec<WarehouseConsumptionRecord>,
}

/// Private interpreter input; cursor values are data and receive adapter literal rendering.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ use crate::platform::models::{LocationHealthReport, LocationWorkClient};
use crate::warehouse_metadata::models::{PreviewCopy, PreviewCopyRead};

const NODE: &str = "preview_copy_orders";
/// A relation with more executions than a build reads recently; read-back can still need them all.
const BUSY_NODE: &str = "preview_copy_customers";

async fn post(client: &Client, (url, bearer): (&str, &str), body: Value) {
let response = client
Expand Down Expand Up @@ -106,6 +108,20 @@ pub(crate) async fn preview_copy_scope(
),
)
.await;
let mut busy: Vec<Value> = Vec::new();
for execution in 1..=10 {
for (phase, hour, minute) in [("started", execution, 0), ("succeeded", execution, 30)] {
let mut row = consumption(
&format!("busy-execution-{execution:02}"),
(phase, &format!("busy-run-{execution:02}")),
&format!("2026-03-02T{hour:02}:{minute:02}:00Z"),
);
row["node_name"] = json!(BUSY_NODE);
row["target_name"] = json!(BUSY_NODE);
busy.push(row);
}
}
post(client, (&metadata, bearer), batch("consumption", busy)).await;
post(
client,
(&metadata, bearer),
Expand Down Expand Up @@ -142,6 +158,7 @@ pub(crate) async fn preview_copy_scope(
let read = PreviewCopyRead {
session_id: preview_session,
generation: 1,
consumption: false,
};
let copy = || async {
outbound
Expand Down Expand Up @@ -174,6 +191,46 @@ pub(crate) async fn preview_copy_scope(
)
};
let unsynced = copy().await;
let older_location: Value = client
.post(format!("{metadata}/preview-copy"))
.bearer_auth(bearer)
.json(&json!({"session_id": preview_session, "generation": 1}))
.send()
.await
.expect("copy for an older location")
.json()
.await
.expect("copy body");
assert!(
unsynced.consumption.is_empty() && older_location.get("consumption").is_none(),
"a read that does not ask for consumption never carries the field, which older locations reject"
);
let with_consumption = outbound
.preview_copy(
"preview",
&PreviewCopyRead {
consumption: true,
..read.clone()
},
)
.await
.expect("preview copy with consumption");
let busy: Vec<(String, String)> = with_consumption
.consumption
.iter()
.filter(|row| row.target_name.as_deref() == Some(BUSY_NODE))
.map(|row| {
(
row.execution_id.clone().unwrap_or_default(),
row.phase.clone().unwrap_or_default(),
)
})
.collect();
assert_eq!(
busy.len(),
20,
"every mirrored row, beyond the newest eight executions, so read-back finds older ones: {busy:?}"
);
assert_eq!(unsynced.synced_at, None, "nothing synced cleanly yet");
assert_eq!(
ours(&unsynced),
Expand Down Expand Up @@ -267,6 +324,7 @@ pub(crate) async fn preview_copy_scope(
&PreviewCopyRead {
session_id,
generation: 1,
consumption: false,
},
)
.await
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use sqlbuild_state::consumption::constants::{
};
use sqlbuild_state::consumption::main::executions_by_id_sql::executions_by_id_sql;
use sqlbuild_state::consumption::main::prune_executions_sql::prune_executions_sql;
use sqlbuild_state::consumption::main::recent_executions::recent_executions;
use sqlbuild_state::consumption::main::recent_executions_sql::recent_executions_sql;
use sqlbuild_state::consumption::models::{
ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey,
Expand All @@ -22,6 +23,7 @@ use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql;

use crate::bindings::_helpers::boundary::python_calls::call_with;
use crate::bindings::_helpers::contracts::state_literals::render_state_sql_literal;
use crate::bindings::_helpers::native_host::synced_copy::SyncedCopyAdapter;
use crate::bindings::_helpers::records::fields::field as native_field;
use crate::bindings::_helpers::state_rows::state_tables::state_table_create_sql;

Expand Down Expand Up @@ -452,6 +454,14 @@ impl ConsumptionConnection {
py: Python<'_>,
location: &Location,
) -> PyResult<Vec<ConsumptionRow>> {
if let Ok(copy) = self.adapter.bind(py).downcast::<SyncedCopyAdapter>() {
return Ok(recent_executions(
&copy
.get()
.consumption((location.database.as_deref(), &location.schema)),
RECENT_EXECUTIONS,
));
}
read_recent(self.adapter.bind(py), &self.connected(py)?, location)
}

Expand All @@ -461,6 +471,13 @@ impl ConsumptionConnection {
location: &Location,
execution_ids: &[&str],
) -> PyResult<Vec<ConsumptionRow>> {
if let Ok(copy) = self.adapter.bind(py).downcast::<SyncedCopyAdapter>() {
let mut rows: Vec<ConsumptionRow> = copy
.get()
.consumption((location.database.as_deref(), &location.schema));
rows.retain(|row| execution_ids.contains(&row.row.execution_id.as_str()));
return Ok(rows);
}
read_executions(
self.adapter.bind(py),
&self.connected(py)?,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ use sqlbuild_cache::digest::main::sha256_hex::sha256_hex;
use sqlbuild_core::constants::{FINGERPRINT_TABLE_NAME, STATE_TABLE_NAMES};
use sqlbuild_host::entries::models::AdapterHost;
use sqlbuild_orchestration::warehouse_metadata::models::{PreviewCopy, WarehouseCatalogRecord};
use sqlbuild_state::consumption::models::{
ConsumedInput, ConsumptionAppend, ConsumptionRow, NodeKey, RelationKey,
};
use sqlbuild_state::execution_history::constants::MEMORY_PATH;
use sqlbuild_state::observations::constants::{CURSOR_MAX_KIND, SOURCE_COLUMNS_KIND};

Expand Down Expand Up @@ -250,6 +253,49 @@ impl SyncedCopyAdapter {
None
}

/// Every copied consumption row of the table in `(database, schema)`.
pub(crate) fn consumption(
&self,
(database, schema): (Option<&str>, &str),
) -> Vec<ConsumptionRow> {
let mut rows: Vec<ConsumptionRow> = Vec::new();
for record in &self.copy.consumption {
if !same(Some(&record.state_schema), Some(schema))
|| !(database.is_none()
|| record.state_database.is_none()
|| same(record.state_database.as_deref(), database))
{
continue;
}
let text = |value: &Option<String>| value.clone().unwrap_or_default();
rows.push(ConsumptionRow {
row: ConsumptionAppend {
execution_id: text(&record.execution_id),
phase: text(&record.phase),
run_id: text(&record.run_id),
node: NodeKey {
kind: text(&record.node_type),
name: text(&record.node_name),
},
relation: RelationKey {
database: record.target_database.clone(),
schema: text(&record.target_schema),
name: text(&record.target_name),
},
version_hash: text(&record.version_hash),
input_count: record.input_count.unwrap_or(-1),
input: record.input_kind.clone().map(|kind| ConsumedInput {
kind,
name: text(&record.input_name),
version: text(&record.input_version),
}),
},
recorded_at: record.recorded_at.timestamp_micros(),
});
}
rows
}

/// The source's columns as last described, only when it described exactly `sql`.
pub(crate) fn described_columns(
&self,
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-state/src/consumption/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub mod consumption_history;
pub mod executions_by_id_sql;
pub mod prune_executions_sql;
pub mod read_back_history;
pub mod recent_executions;
pub mod recent_executions_sql;
pub mod started_append;
pub mod succeeded_appends;
37 changes: 37 additions & 0 deletions crates/sqlbuild-state/src/consumption/main/recent_executions.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
//! Choose the recent executions of every relation from rows already read.

use std::collections::{BTreeMap, BTreeSet};

use crate::consumption::models::{ConsumptionRow, RelationKey};

/// Rows of the newest `limit` executions per relation by earliest row, as `recent_executions_sql`.
pub fn recent_executions(rows: &[ConsumptionRow], limit: usize) -> Vec<ConsumptionRow> {
let mut started: BTreeMap<(&RelationKey, &str), i64> = BTreeMap::new();
for row in rows {
let entry = started
.entry((&row.row.relation, row.row.execution_id.as_str()))
.or_insert(row.recorded_at);
*entry = (*entry).min(row.recorded_at);
}
let mut by_relation: BTreeMap<&RelationKey, Vec<(i64, &str)>> = BTreeMap::new();
for ((relation, execution), time) in started {
by_relation
.entry(relation)
.or_default()
.push((time, execution));
}
let mut kept: BTreeSet<(&RelationKey, &str)> = BTreeSet::new();
for (relation, executions) in &mut by_relation {
executions.sort_by(|left, right| right.cmp(left));
kept.extend(
executions
.iter()
.take(limit)
.map(|(_, execution)| (*relation, *execution)),
);
}
rows.iter()
.filter(|row| kept.contains(&(&row.row.relation, row.row.execution_id.as_str())))
.cloned()
.collect()
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-state/src/consumption/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
mod helpers;
#[path = "tests/test_decide.rs"]
mod test_decide;
#[path = "tests/test_recent_executions.rs"]
mod test_recent_executions;
#[path = "tests/test_simulation.rs"]
mod test_simulation;
#[path = "tests/test_types.rs"]
Expand Down
Loading
Loading