Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,9 +265,9 @@ fn missing_column(
format!(
"add {name} to the SELECT list or correct/remove {declaration_label}; \
{declaration_label} is validated using static SQL analysis because \
settings.column_contract_mode is \"implicit\" (the default). If this project \
settings.check_declared_columns is true (the default). If this project \
intentionally uses columns only for metadata and audits, set [settings] \
column_contract_mode = \"explicit\"; models with contract enforced remain validated"
check_declared_columns = false; models with contract enforced remain validated"
)
};
ContractDiagnostic {
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-analysis/src/contracts/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
pub struct ContractRequest {
/// The dialect name Python hands type normalization (`str(dialect or "generic")`).
pub dialect: String,
/// Whether `settings.column_contract_mode` is `implicit`.
/// Whether `settings.check_declared_columns` is on.
pub implicit_column_contracts: bool,
/// Whether the effective table promotion mode is `staged`.
pub staged_promotion: bool,
Expand Down
6 changes: 1 addition & 5 deletions crates/sqlbuild-cache/src/hosted_shapes/constants.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! The Python shape names of the natively hosted compiler shapes.

/// The Python model type a natively hosted compiler shape is captured as.
pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [
pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [
(
"sqlbuild._native:AlignedInterval",
"sqlbuild.cursor_algebra.models:AlignedInterval",
Expand Down Expand Up @@ -62,10 +62,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [
"sqlbuild._native:CollectionRendering",
"sqlbuild.sql_values.types:CollectionRendering",
),
(
"sqlbuild._native:ColumnContractMode",
"sqlbuild.spec.contracts.types:ColumnContractMode",
),
(
"sqlbuild._native:ColumnLineage",
"sqlbuild.compiler.lineage.models:ColumnLineage",
Expand Down
17 changes: 3 additions & 14 deletions crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@
use std::path::Path;

use sqlbuild_config::project::main::load_project_config::load_project_config;
use sqlbuild_config::project::main::project_config_paths::project_config_paths;
use sqlbuild_config::project::models::{Built, ConfigPaths};
use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME;
use sqlbuild_orchestration::store::errors::ControlError;

use crate::native_commands::main::effective_target::effective_target;

use crate::native_commands::constants::{
CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME,
LEGACY_PROJECT_CONFIG_FILENAME, LIFECYCLE_SINK_PATH, LOCAL_CONFIG_FILENAME, SINKS_DIRECTORY,
CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LIFECYCLE_SINK_PATH, SINKS_DIRECTORY,
};

pub(crate) fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> {
Expand All @@ -36,24 +35,14 @@ fn names_lifecycle_sinks(config: &Built) -> bool {
}
}

/// The four configuration files of `project_dir`.
pub(crate) fn config_paths(project_dir: &Path) -> ConfigPaths {
ConfigPaths {
project: project_dir.join(PROJECT_CONFIG_FILENAME),
legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME),
local: project_dir.join(LOCAL_CONFIG_FILENAME),
legacy_local: project_dir.join(LEGACY_LOCAL_CONFIG_FILENAME),
}
}

/// Whether `discover_runtime_extensions` would import nothing and accept the project file.
pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool {
if project_dir.join(EVENT_EXPORTERS_DIRECTORY).exists()
|| project_dir.join(SINKS_DIRECTORY).is_dir()
{
return false;
}
let paths: ConfigPaths = config_paths(project_dir);
let paths: ConfigPaths = project_config_paths(project_dir);
if !paths.project.exists() {
return true;
}
Expand Down
3 changes: 0 additions & 3 deletions crates/sqlbuild-cli/src/native_commands/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,6 @@ pub(crate) const SKILL_FILES_HEADER: &str = "Updated SQLBuild skill files:\n";
pub(crate) const RULE_SELECTOR_REQUIRED: &str = "A rule code or family prefix is required";
/// `run_rules_command`'s exit status for a missing or unknown rule.
pub(crate) const RULES_USAGE_EXIT_CODE: i32 = 2;
pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml";
pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml";
pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml";
/// The class Python's last-resort log handler names for a user error.
pub(crate) const USER_ERROR_CLASS: &str = "sqlbuild._native.CliUserError";
pub(crate) const INIT_FAILURE_PREFIX: &str = "sqb init: ";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,17 @@ use sqlbuild_config::effective::models::TargetChoice;
use sqlbuild_config::models::ConfigValue;
use sqlbuild_config::project::main::load_local_config::load_local_config;
use sqlbuild_config::project::main::load_project_config::load_project_config;
use sqlbuild_config::project::main::project_config_paths::project_config_paths;
use sqlbuild_config::project::models::Built;

use crate::native_commands::_helpers::project_facts::{config_paths, object_field};
use crate::native_commands::_helpers::project_facts::object_field;

/// The effective target of `project_dir` with `selected` as `--target`, `None` when nothing selects one.
pub fn effective_target(
project_dir: &Path,
selected: Option<String>,
) -> Result<Option<String>, String> {
let paths = config_paths(project_dir);
let paths = project_config_paths(project_dir);
let invalid = |_| "project configuration is invalid; run sqb debug".to_owned();
let project = load_project_config(&paths).map_err(invalid)?;
let local = load_local_config(&paths).map_err(invalid)?;
Expand Down
11 changes: 9 additions & 2 deletions crates/sqlbuild-cli/src/replay/_helpers/serving.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
//! The native reuse check against a launch record and the output a hit replays.

use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::time::Instant;

use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse;
use sqlbuild_cache::project_reuse::main::replay_stdout::replay_stdout;
use sqlbuild_cache::project_reuse::models::{ReuseCheck, ReuseOutcome, ReuseRules, StoredOutput};
use sqlbuild_config::project::main::deprecated_setting_warnings::deprecated_setting_warnings;
use sqlbuild_config::project::main::project_config_paths::project_config_paths;

use crate::replay::_helpers::console::tenths_of_seconds;
use crate::replay::_helpers::records::{hit_timings, host_identity};
Expand Down Expand Up @@ -49,7 +51,12 @@ pub(crate) fn served(
let stdout = replay_stdout(output, &timings)?;
let (prefix, suffix) = &record.hit_message;
let mut stderr = format!("{prefix}{}{suffix}\n", tenths_of_seconds(check_ms));
for line in &output.stderr_lines {
let warnings: Vec<String> =
match deprecated_setting_warnings(&project_config_paths(Path::new(&record.project_dir))) {
Ok(warnings) => warnings,
Err(_) => return None,
};
for line in warnings.iter().chain(&output.stderr_lines) {
stderr.push_str(line);
stderr.push('\n');
}
Expand Down
37 changes: 35 additions & 2 deletions crates/sqlbuild-config/src/project/_helpers/documents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ use crate::project::_helpers::targets::{
};
use crate::project::_helpers::values::is_truthy;
use crate::project::constants::{
LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY,
REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY,
COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_EVENT_EXPORTERS_KEY,
LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY,
};
use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece};
use crate::project::models::{Built, ConfigFile, ConfigPaths, ReviewConfig};
Expand Down Expand Up @@ -441,3 +441,36 @@ pub(crate) fn review_config(paths: &ConfigPaths) -> Result<ReviewConfig, LoadFai
let payload = table.as_map().unwrap_or_default();
Ok(load_review(get(payload, "review"), PROJECT_FILE)?)
}

/// One warning line per configuration file whose `[settings]` still sets `column_contract_mode`.
pub(crate) fn deprecated_setting_warnings(paths: &ConfigPaths) -> Result<Vec<String>, LoadFailure> {
let mut warnings: Vec<String> = Vec::new();
for (local, name) in [
(false, PROJECT_CONFIG_FILENAME),
(true, LOCAL_CONFIG_FILENAME),
] {
let file = resolve(paths, local)?;
if !path_of(paths, file).exists() {
continue;
}
if let Some(checked) = legacy_column_contract_mode(&load_toml_mapping(paths, file)?) {
warnings.push(format!(
"Warning: settings.column_contract_mode in {name} is deprecated; replace it with \
settings.check_declared_columns = {checked}"
));
}
}
Ok(warnings)
}

/// The `check_declared_columns` value a valid `[settings] column_contract_mode` alias maps to.
fn legacy_column_contract_mode(table: &ConfigValue) -> Option<bool> {
let settings = get(table.as_map()?, SETTINGS_SECTION)?.as_map()?;
let ConfigValue::String(mode) = get(settings, LEGACY_COLUMN_CONTRACT_MODE_KEY)? else {
return None;
};
COLUMN_CONTRACT_MODES
.iter()
.find(|(value, _)| value == mode)
.map(|(_, checked)| *checked)
}
68 changes: 45 additions & 23 deletions crates/sqlbuild-config/src/project/_helpers/sections.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,16 @@ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validat
use crate::project::_helpers::targets::seconds_value;
use crate::project::_helpers::values::{int_below, python_equal};
use crate::project::constants::{
AUDIT_SEVERITIES, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY,
DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_IN_USE_WINDOW,
DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY,
LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS,
OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY,
REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH,
SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS,
SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY,
UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT,
AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES,
CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS,
DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS,
LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY,
LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY,
QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS,
RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS,
SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS,
SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES,
USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT,
};
use crate::project::constants::{
DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE,
Expand Down Expand Up @@ -101,14 +102,24 @@ fn settings_mapping(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded<&
)?;
let allowed: Vec<&str> = SETTINGS_KEYS
.into_iter()
.chain([LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY])
.chain([
LEGACY_CONCURRENCY_KEY,
LEGACY_SQL_VALIDATION_KEY,
LEGACY_COLUMN_CONTRACT_MODE_KEY,
])
.collect();
validate_allowed_keys(mapping, &allowed, place(file, "settings"))?;
if has(mapping, CONCURRENCY_KEY) && has(mapping, LEGACY_CONCURRENCY_KEY) {
return Err(ConfigFailure::plain(
"settings cannot define both 'concurrency' and legacy 'max_concurrency'",
));
}
if has(mapping, CHECK_DECLARED_COLUMNS_KEY) && has(mapping, LEGACY_COLUMN_CONTRACT_MODE_KEY) {
return Err(ConfigFailure::plain(
"settings cannot define both 'check_declared_columns' and deprecated \
'column_contract_mode'; keep only check_declared_columns",
));
}
if let (Some(current), Some(legacy)) = (
get(mapping, SQL_ANALYSIS_KEY),
get(mapping, LEGACY_SQL_VALIDATION_KEY),
Expand All @@ -121,20 +132,27 @@ fn settings_mapping(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded<&
Ok(mapping)
}

fn column_contract_mode(mapping: &Entries) -> Loaded<Built> {
let mode = match get(mapping, "column_contract_mode") {
None => Some(COLUMN_CONTRACT_MODES[0]),
/// `check_declared_columns`, or what the deprecated `column_contract_mode` it replaces means.
fn check_declared_columns(mapping: &Entries) -> Loaded<bool> {
if !has(mapping, LEGACY_COLUMN_CONTRACT_MODE_KEY) {
return optional_bool(mapping, CHECK_DECLARED_COLUMNS_KEY, true);
}
let mode = match get(mapping, LEGACY_COLUMN_CONTRACT_MODE_KEY) {
Some(ConfigValue::String(text)) => Some(text.as_str()),
Some(_) => None,
_ => None,
};
match mode {
Some(text) if COLUMN_CONTRACT_MODES.contains(&text) => {
Ok(member("ColumnContractMode", text))
}
_ => Err(ConfigFailure::plain(
"settings.column_contract_mode must be one of: implicit, explicit",
)),
}
COLUMN_CONTRACT_MODES
.iter()
.find(|(name, _)| Some(*name) == mode)
.map(|(_, checked)| *checked)
.ok_or_else(|| {
ConfigFailure::plain(
"settings.column_contract_mode must be one of: implicit, explicit. It is \
deprecated; set this in sqlbuild_project.toml instead, with false to leave \
declared columns unchecked:\n [settings]\n \
check_declared_columns = true",
)
})
}

fn unaccounted_partition_policy(mapping: &Entries) -> Loaded<String> {
Expand Down Expand Up @@ -208,7 +226,10 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) ->
true,
)?)),
),
("column_contract_mode", column_contract_mode(mapping)?),
(
"check_declared_columns",
Built::Value(ConfigValue::Bool(check_declared_columns(mapping)?)),
),
(
"auto_load_sources",
Built::Value(ConfigValue::Bool(optional_bool(
Expand Down Expand Up @@ -275,6 +296,7 @@ pub(crate) fn setting_overrides(mapping: &Entries) -> Vec<String> {
let name = match key_name(key) {
LEGACY_CONCURRENCY_KEY => CONCURRENCY_KEY,
LEGACY_SQL_VALIDATION_KEY => SQL_ANALYSIS_KEY,
LEGACY_COLUMN_CONTRACT_MODE_KEY => CHECK_DECLARED_COLUMNS_KEY,
other if SETTINGS_KEYS.contains(&other) => other,
_ => continue,
};
Expand Down
11 changes: 9 additions & 2 deletions crates/sqlbuild-config/src/project/constants.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
//! Project configuration file names, keys and the vocabularies their values come from.

pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml";
pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml";
pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml";
pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis";
pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation";
pub(crate) const CHECK_DECLARED_COLUMNS_KEY: &str = "check_declared_columns";
/// The deprecated alias of `check_declared_columns`.
pub(crate) const LEGACY_COLUMN_CONTRACT_MODE_KEY: &str = "column_contract_mode";
pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis";
pub(crate) const QUERY_CHANGE_TRACKING_KEY: &str = "query_change_tracking";
pub(crate) const CONCURRENCY_KEY: &str = "concurrency";
Expand Down Expand Up @@ -48,7 +53,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 15] = [
"sql_analysis",
"require_sql_analysis",
"query_change_tracking",
"column_contract_mode",
"check_declared_columns",
"concurrency",
"auto_load_sources",
"microbatch_concurrency",
Expand Down Expand Up @@ -253,7 +258,9 @@ pub(crate) const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] =
["append_new_columns", "deny", "require_confirmation"];
pub(crate) const UNACCOUNTED_PARTITION_POLICIES: [&str; 3] =
["recover_all", "recover_empty", "synthesize"];
pub(crate) const COLUMN_CONTRACT_MODES: [&str; 2] = ["implicit", "explicit"];
/// `column_contract_mode` values and the `check_declared_columns` value each means.
pub(crate) const COLUMN_CONTRACT_MODES: [(&str, bool); 2] =
[("implicit", true), ("explicit", false)];
pub(crate) const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"];
pub(crate) const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"];
pub(crate) const FUTURE_CURSOR_ACTIONS: [&str; 2] = ["cap", "error"];
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! Warnings for deprecated `[settings]` keys that still load as aliases.

use crate::project::errors::LoadFailure;
use crate::project::models::ConfigPaths;

/// One `Warning:` line per configuration file that sets a deprecated setting alias.
pub fn deprecated_setting_warnings(paths: &ConfigPaths) -> Result<Vec<String>, LoadFailure> {
crate::project::_helpers::documents::deprecated_setting_warnings(paths)
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-config/src/project/main/mod.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
pub mod deprecated_setting_warnings;
pub mod explicit_references_help;
pub mod load_git_base_ref;
pub mod load_local_config;
pub mod load_project_config;
pub mod load_review_config;
pub mod local_sql_analysis_failure;
pub mod microbatch_guidance;
pub mod project_config_paths;
pub mod resolve_config_file;
pub mod setting_help_text;
pub mod snippet_help_text;
Expand Down
20 changes: 20 additions & 0 deletions crates/sqlbuild-config/src/project/main/project_config_paths.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
//! The configuration files a project directory may hold.

use std::path::Path;

use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME;

use crate::project::constants::{
LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_PROJECT_CONFIG_FILENAME, LOCAL_CONFIG_FILENAME,
};
use crate::project::models::ConfigPaths;

/// The four configuration files of `project_dir`.
pub fn project_config_paths(project_dir: &Path) -> ConfigPaths {
ConfigPaths {
project: project_dir.join(PROJECT_CONFIG_FILENAME),
legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME),
local: project_dir.join(LOCAL_CONFIG_FILENAME),
legacy_local: project_dir.join(LEGACY_LOCAL_CONFIG_FILENAME),
}
}
Loading
Loading