Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/sqlbuild-config/src/command_connection/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ pub(crate) const UNSUPPORTED_WAREHOUSE_CODE: &str = "C260";
pub(crate) const SETTING_SNIPPET_INDENT: &str = " ";
/// Two quotes around at least one character.
pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3;
/// The warehouse group each CLI command selects a target's default warehouse from.
/// The warehouse group each CLI command uses; a location's metadata syncs use the cheap `sync` group.
pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [
("audit", Some("build")),
("build", Some("build")),
Expand All @@ -44,13 +44,13 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [
("janitor", Some("query")),
("scenario", Some("query")),
("test", Some("query")),
("location", Some("sync")),
("clean", None),
("compile", None),
("dag", None),
("format", None),
("init", None),
("lineage", None),
("location", None),
("mv", None),
("playground", None),
("rename", None),
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-config/src/project/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ pub(crate) const EVENT_EXPORT_SEVERITIES: [&str; 4] = ["debug", "info", "warning
/// `TIME_TRAVEL_RETENTION_MATERIALIZATIONS` in declared order.
pub(crate) const RETENTION_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"];
pub(crate) const MATERIALIZATION_DEFAULT_KEYS: [&str; 2] = ["time_travel_retention", "table_type"];
pub(crate) const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"];
pub(crate) const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"];
pub(crate) const SNAPSHOT_FULL_REFRESH_POLICIES: [&str; 3] =
["allow", "deny", "require_confirmation"];
pub(crate) const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] =
Expand Down
9 changes: 9 additions & 0 deletions crates/sqlbuild-core/src/cost_command/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,12 @@ pub const MISSING_METRIC_STATUSES: [&str; 4] =
/// Statuses a delayed refresh may still improve.
pub const REFRESHABLE_STATUSES: [&str; 2] = ["pending", "partial"];
pub const LEDGER_PERSISTENCE_FAILURE_PREFIX: &str = "Statement-ledger persistence failed (";
/// The limitation of a run whose statements query history still showed running, which a re-read can complete.
pub const INCOMPLETE_HISTORY_LIMITATION: &str =
"One or more run statements were still incomplete in query history.";
/// The limitation of a run whose recent statements had no warehouse metadata yet, which a re-read can complete.
pub const MISSING_METADATA_LIMITATION: &str =
"One or more run statements had missing Snowflake warehouse metadata.";
/// The limitations a re-read of a run that saw every statement can still lift.
pub const CHANGEABLE_LIMITATIONS: [&str; 2] =
[INCOMPLETE_HISTORY_LIMITATION, MISSING_METADATA_LIMITATION];
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ const E_EVENT_EXPORT_SEVERITY: EnumSpec = EnumSpec {
const E_WAREHOUSE_GROUP: EnumSpec = EnumSpec {
name: "WarehouseGroup",
doc: "Command group that selects a target's default warehouse.",
members: &[("BUILD", "build"), ("QUERY", "query")],
members: &[("BUILD", "build"), ("QUERY", "query"), ("SYNC", "sync")],
};
const E_SOURCE_WRITE_STRATEGY: EnumSpec = EnumSpec {
name: "SourceWriteStrategy",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ const R_TARGET_WAREHOUSES_CONFIG: RecordSpec = RecordSpec {
fields: &[
FieldSpec::new("build", FieldDefault::None),
FieldSpec::new("query", FieldDefault::None),
FieldSpec::new("sync", FieldDefault::None),
],
};
const R_TARGET_CONFIG: RecordSpec = RecordSpec {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use sqlbuild_python_base::records::types::ErrorId;
const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml";
const SQL_ANALYSIS_KEY: &str = "sql_analysis";
/// `WarehouseGroup` members in declaration order.
const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"];
const WAREHOUSE_GROUPS: [&str; 3] = ["build", "query", "sync"];

fn project_config_error(py: Python<'_>, message: &str, help: Option<&str>) -> PyErr {
new_error(py, ErrorId::ProjectConfig, message, (None, help))
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
//! Elapsed time between two Python datetimes.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};

/// Seconds from `from` to `to`, negative when `to` is earlier.
pub fn elapsed_seconds(from: &Bound<'_, PyAny>, to: &Bound<'_, PyAny>) -> PyResult<f64> {
to.sub(from)?.call_method0("total_seconds")?.extract()
}
3 changes: 3 additions & 0 deletions crates/sqlbuild-python-front/src/native_host/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
pub mod append_observations;
pub mod append_replaced_consumption;
pub mod elapsed_seconds;
pub mod execute;
pub mod fixed_state_table_sql;
pub mod integer_literal;
Expand All @@ -9,4 +10,6 @@ pub mod recorder_callbacks;
pub mod register_audit_results_state;
pub mod register_consumption_state;
pub mod register_observations_state;
pub mod reports_cost_capability;
pub mod seconds_since;
pub mod state_table_name;
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//! Whether a cost-aware adapter reports a cost capability.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_host::entries::models::ExecutorHost;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;

/// Whether `adapter` reports a cost capability; a failing report still counts as one.
pub fn reports_cost_capability(adapter: &Bound<'_, PyAny>) -> PyResult<bool> {
let none: Bound<'_, PyAny> =
host_entry(adapter.py(), ExecutorHost::CostCapability)?.getattr("NONE")?;
Ok(match adapter.call_method0("cost_capability") {
Ok(capability) => !capability.eq(none)?,
Err(_) => true,
})
}
14 changes: 14 additions & 0 deletions crates/sqlbuild-python-front/src/native_host/main/seconds_since.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
//! Time since a Python datetime, by the UTC clock now.

use crate::cursor::constants::DATETIME_MODULE;
use crate::native_host::main::elapsed_seconds::elapsed_seconds;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use sqlbuild_python_base::boundary::main::imported::imported;

/// Seconds since `value`, by the UTC clock now.
pub fn seconds_since(value: &Bound<'_, PyAny>) -> PyResult<f64> {
let py: Python<'_> = value.py();
let now: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "datetime")?
.call_method1("now", (imported(py, DATETIME_MODULE, "UTC")?,))?;
elapsed_seconds(value, &now)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
//! Plain cost runs read from `CostRunRecord` objects, and their fields built back into Python values.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{PyDict, PyDictMethods, PyTuple};
use sqlbuild_core::cost_report::models::{CostResource, CostSummary};
use sqlbuild_host::entries::models::ExecutorHost;
use sqlbuild_python_base::boundary::main::imported::imported;
use sqlbuild_python_base::native_host::main::host_entry::host_entry;
use sqlbuild_python_base::records::main::construct::new_record::new_record;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_base::records::types::RecordId;

pub(crate) fn text(value: &Bound<'_, PyAny>, field: &str) -> PyResult<String> {
native_field(value, field)?.extract()
}

pub(crate) fn optional_text(value: &Bound<'_, PyAny>, field: &str) -> PyResult<Option<String>> {
native_field(value, field)?.extract()
}

/// `str(value.<field>)`, the text a `Decimal` field is stored as.
pub(crate) fn decimal(value: &Bound<'_, PyAny>, field: &str) -> PyResult<String> {
Ok(native_field(value, field)?.str()?.to_string())
}

pub(crate) fn integer(value: &Bound<'_, PyAny>, field: &str) -> PyResult<i64> {
native_field(value, field)?.extract()
}

pub(crate) fn timestamp(value: &Bound<'_, PyAny>, field: &str) -> PyResult<String> {
native_field(value, field)?
.call_method0("isoformat")?
.extract()
}

pub(crate) fn resource(value: &Bound<'_, PyAny>) -> PyResult<CostResource> {
Ok(CostResource {
resource_type: text(value, "resource_type")?,
resource_name: text(value, "resource_name")?,
warehouse_name: text(value, "warehouse_name")?,
warehouse_size: text(value, "warehouse_size")?,
warehouse_type: optional_text(value, "warehouse_type")?,
cluster_number: native_field(value, "cluster_number")?.extract()?,
query_count: integer(value, "query_count")?,
attributed_seconds: decimal(value, "attributed_seconds")?,
bytes_scanned: integer(value, "bytes_scanned")?,
estimated_compute_credits: decimal(value, "estimated_compute_credits")?,
estimated_usd: decimal(value, "estimated_usd")?,
})
}

/// Python constructors of the record fields.
pub(crate) struct Builders<'py> {
pub(crate) decimal: Bound<'py, PyAny>,
pub(crate) timestamp: Bound<'py, PyAny>,
}

impl<'py> Builders<'py> {
pub(crate) fn new(py: Python<'py>) -> PyResult<Self> {
Ok(Self {
decimal: imported(py, "decimal", "Decimal")?,
timestamp: imported(py, "datetime", "datetime")?.getattr("fromisoformat")?,
})
}

pub(crate) fn decimal(&self, text: &str) -> PyResult<Bound<'py, PyAny>> {
self.decimal.call1((text,))
}
}

pub(crate) fn resource_object<'py>(
builders: &Builders<'py>,
resource: &CostResource,
) -> PyResult<Bound<'py, PyAny>> {
let py: Python<'py> = builders.decimal.py();
let fields: Bound<'py, PyDict> = PyDict::new(py);
fields.set_item("resource_type", &resource.resource_type)?;
fields.set_item("resource_name", &resource.resource_name)?;
fields.set_item("warehouse_name", &resource.warehouse_name)?;
fields.set_item("warehouse_size", &resource.warehouse_size)?;
fields.set_item("warehouse_type", &resource.warehouse_type)?;
fields.set_item("cluster_number", resource.cluster_number)?;
fields.set_item("query_count", resource.query_count)?;
fields.set_item(
"attributed_seconds",
builders.decimal(&resource.attributed_seconds)?,
)?;
fields.set_item("bytes_scanned", resource.bytes_scanned)?;
fields.set_item(
"estimated_compute_credits",
builders.decimal(&resource.estimated_compute_credits)?,
)?;
fields.set_item("estimated_usd", builders.decimal(&resource.estimated_usd)?)?;
new_record(py, RecordId::ResourceCost, (), Some(&fields))
}

pub(crate) fn summary_object<'py>(
builders: &Builders<'py>,
cost: &CostSummary,
) -> PyResult<Bound<'py, PyAny>> {
let py: Python<'py> = builders.decimal.py();
let mut resources: Vec<Bound<'py, PyAny>> = Vec::with_capacity(cost.resources.len());
for resource in &cost.resources {
resources.push(resource_object(builders, resource)?);
}
let fields: Bound<'py, PyDict> = PyDict::new(py);
fields.set_item(
"status",
host_entry(py, ExecutorHost::CostStatus)?.call1((cost.status.as_str(),))?,
)?;
fields.set_item("usd_per_credit", builders.decimal(&cost.usd_per_credit)?)?;
fields.set_item("rate_source", &cost.rate_source)?;
fields.set_item("resources", PyTuple::new(py, resources)?)?;
fields.set_item("query_count", cost.query_count)?;
fields.set_item(
"attributed_seconds",
builders.decimal(&cost.attributed_seconds)?,
)?;
fields.set_item("bytes_scanned", cost.bytes_scanned)?;
fields.set_item(
"estimated_compute_credits",
builders.decimal(&cost.estimated_compute_credits)?,
)?;
fields.set_item("estimated_usd", builders.decimal(&cost.estimated_usd)?)?;
fields.set_item("expected_statement_count", cost.expected_statement_count)?;
fields.set_item("observed_statement_count", cost.observed_statement_count)?;
fields.set_item("missing_statement_count", cost.missing_statement_count)?;
fields.set_item("skipped_statement_count", cost.skipped_statement_count)?;
fields.set_item("source", &cost.source)?;
fields.set_item("method", &cost.method)?;
fields.set_item("limitations", PyTuple::new(py, &cost.limitations)?)?;
fields.set_item("message", &cost.message)?;
host_entry(py, ExecutorHost::RunCostSummary)?.call((), Some(&fields))
}
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pub(crate) mod cost_run_records;
pub(crate) mod resource_costs_state;
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field;
const CONFIGURED_RATE_SOURCE: &str = "configured";
/// The adapter method that marks the resource costs table as supported.
pub(crate) const CREATE_TABLE_METHOD: &str = "render_create_resource_costs_table_sql";
/// Collection budget of each re-read of an older run, which a build waits for.
pub(crate) const REFRESH_BUDGET_SECONDS: f64 = 20.0;

/// Create-if-missing DDL for the resource costs table, with adapter-rendered column types.
#[pyfunction]
Expand Down Expand Up @@ -210,9 +208,14 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult<RunCostAttempt> {
let limitations: Vec<String> = serde_json::from_str(&text(7)?)
.map_err(|error| PyValueError::new_err(format!("unreadable limitations: {error}")))?;
let run_started_at: String = utc_text(&cell(9)?)?;
let started_epoch: i64 = DateTime::parse_from_rfc3339(&run_started_at)
.map_err(|error| PyValueError::new_err(error.to_string()))?
.timestamp();
let epoch = |text: &str| -> PyResult<i64> {
Ok(DateTime::parse_from_rfc3339(text)
.map_err(|error| PyValueError::new_err(error.to_string()))?
.timestamp())
};
let started_epoch: i64 = epoch(&run_started_at)?;
let run_completed_at: String = utc_text(&cell(10)?)?;
let completed_epoch: i64 = epoch(&run_completed_at)?;
Ok(RunCostAttempt {
run_id: text(0)?,
orchestration_run_id: optional_text(&cell(1)?)?,
Expand All @@ -226,7 +229,8 @@ pub(crate) fn run_attempt(row: &Bound<'_, PyAny>) -> PyResult<RunCostAttempt> {
},
target_name: optional_text(&cell(8)?)?,
run_started_at,
run_completed_at: utc_text(&cell(10)?)?,
run_completed_at,
started_epoch,
completed_epoch,
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
//! The `CostArtifactError` Python raised for a cost artifact.

use pyo3::prelude::{PyAnyMethods, PyErr, Python};
use sqlbuild_core::cost_report::errors::CostArtifactError;
use sqlbuild_python_base::records::main::registry::native_attr::native_attr;
use sqlbuild_python_base::records::types::NativeAttr;

/// The `CostArtifactError` Python raised for the same artifact.
pub fn artifact_error(py: Python<'_>, error: &CostArtifactError) -> PyErr {
match native_attr(py, NativeAttr::CostArtifactError)
.and_then(|class| class.call1((error.message.as_str(),)))
{
Ok(instance) => PyErr::from_value(instance),
Err(failure) => failure,
}
}
25 changes: 25 additions & 0 deletions crates/sqlbuild-python-front/src/resource_costs/main/cost_run.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//! A `CostRunRecord` as a plain run.

use crate::resource_costs::_helpers::cost_run_records::integer;
use crate::resource_costs::_helpers::cost_run_records::optional_text;
use crate::resource_costs::_helpers::cost_run_records::text;
use crate::resource_costs::_helpers::cost_run_records::timestamp;
use crate::resource_costs::main::summary::summary;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_core::cost_report::models::CostRun;
use sqlbuild_python_base::records::main::fields::field::field as native_field;

/// A `CostRunRecord` as a plain run.
pub fn cost_run(record: &Bound<'_, PyAny>) -> PyResult<CostRun> {
Ok(CostRun {
run_id: text(record, "run_id")?,
adapter_name: text(record, "adapter_name")?,
target_name: optional_text(record, "target_name")?,
build_status: text(record, "build_status")?,
started_at: timestamp(record, "started_at")?,
completed_at: timestamp(record, "completed_at")?,
cost: summary(&native_field(record, "cost")?)?,
version: integer(record, "version")?,
had_executable_work: native_field(record, "had_executable_work")?.extract()?,
})
}
14 changes: 14 additions & 0 deletions crates/sqlbuild-python-front/src/resource_costs/main/cost_runs.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
//! Every `CostRunRecord` of an iterable as a plain run.

use crate::resource_costs::main::cost_run::cost_run;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult};
use sqlbuild_core::cost_report::models::CostRun;

/// Every record of an iterable as a plain run.
pub fn cost_runs(records: &Bound<'_, PyAny>) -> PyResult<Vec<CostRun>> {
let mut runs: Vec<CostRun> = Vec::new();
for record in records.try_iter()? {
runs.push(cost_run(&record?)?);
}
Ok(runs)
}
6 changes: 6 additions & 0 deletions crates/sqlbuild-python-front/src/resource_costs/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1 +1,7 @@
pub mod artifact_error;
pub mod cost_run;
pub mod cost_runs;
pub mod record_object;
pub mod record_tuple;
pub mod register_resource_costs_state;
pub mod summary;
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
//! A plain run as the `CostRunRecord` it was read from.

use crate::resource_costs::_helpers::cost_run_records::Builders;
use crate::resource_costs::_helpers::cost_run_records::summary_object;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::types::{PyDict, PyDictMethods};
use sqlbuild_core::cost_report::models::CostRun;
use sqlbuild_python_base::records::main::construct::new_record::new_record;
use sqlbuild_python_base::records::types::RecordId;

/// A plain run as the `CostRunRecord` it was read from.
pub fn record_object<'py>(py: Python<'py>, run: &CostRun) -> PyResult<Bound<'py, PyAny>> {
let builders: Builders<'py> = Builders::new(py)?;
let fields: Bound<'py, PyDict> = PyDict::new(py);
fields.set_item("version", run.version)?;
fields.set_item("run_id", &run.run_id)?;
fields.set_item("adapter_name", &run.adapter_name)?;
fields.set_item("target_name", &run.target_name)?;
fields.set_item("build_status", &run.build_status)?;
fields.set_item(
"started_at",
builders.timestamp.call1((run.started_at.as_str(),))?,
)?;
fields.set_item(
"completed_at",
builders.timestamp.call1((run.completed_at.as_str(),))?,
)?;
fields.set_item("cost", summary_object(&builders, &run.cost)?)?;
fields.set_item("had_executable_work", run.had_executable_work)?;
new_record(py, RecordId::CostRunRecord, (), Some(&fields))
}
Loading
Loading