diff --git a/Makefile b/Makefile index d45aa2ba11..1b5cb187ea 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ +.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-parse-guard check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ check-e2e-shards check-test-shards test-shard-core test-shard-publication test-shard-history \ test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ @@ -747,7 +747,7 @@ verify-quick: exit $$status -check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint +check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-parse-guard check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests @@ -770,6 +770,10 @@ check-native-stub: uv run python -m scripts.check_native_stub +check-parse-guard: + uv run python -m scripts.check_parse_guard + + check-build-features: uv run python -m scripts.check_build_features diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs index 514da50e62..adc88494de 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs @@ -5,7 +5,7 @@ use std::num::IntErrorKind; use sqlquack_parse::{StrictFailure, parse_strict}; use sqlquack_semantics::dialects::constants::{SNOWFLAKE, SNOWFLAKE_QUOTED_IGNORING_CASE}; use sqlquack_semantics::dialects::types::DialectProfile; -use sqlquack_syntax::{Dialect, Error, Expression, ParseOptions, Parsed, TokenizeFailure}; +use sqlquack_syntax::{Dialect, Error, Expression, Parsed, TokenizeFailure}; use crate::analysis_route::_helpers::dialects::engine_types::snowflake_engine_type; use crate::analysis_route::_helpers::dialects::type_spelling::respelt; @@ -14,6 +14,7 @@ use crate::analysis_route::constants::{ }; use crate::analysis_route::models::{AnalysisInput, ParserGap, SnowflakeDialect}; use crate::analysis_route::types::NativeDialect; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::models::{NormalizedType, TypeDialect, TypeFamily}; @@ -67,7 +68,7 @@ impl NativeDialect for SnowflakeDialect { /// Every statement the shared parser reads from `sql`, or its first refusal, never marked syntax. pub(crate) fn shared_parsed(sql: &str, dialect: Dialect) -> Result { - let statements: Vec = parse_strict(sql, dialect, &ParseOptions::default()) + let statements: Vec = parse_strict(sql, dialect, &sqlquack_parse_options()) .map_err(|failure| match failure { StrictFailure::Tokens( TokenizeFailure::Uncovered { start, .. } | TokenizeFailure::Overlap { start, .. }, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs index 9602a08aab..155cb4940b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs @@ -2,20 +2,20 @@ use polyglot_sql::expressions::Cte; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions, ast_json}; -use serde_json::{Map, Value, json}; +use polyglot_sql::{DataType, Dialect, Expression, ParseOptions, ast_json}; +use serde_json::{Map, Value}; use crate::assembly::analysis_session::models::ProjectionFacts; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::relation_markers::constants::MARKER_NAME_PREFIX; use crate::assembly::analysis_session::constants::{ ALIAS_AST_KIND, ANNOTATED_AST_KIND, ARRAY_FIXTURE_TYPE, ARRAY_PAYLOAD_KIND, CATALOG_TABLE_ARG, COLUMN_AST_KIND, CTE_ALIAS_PAYLOAD_KEY, CTE_LIST_PAYLOAD_KEY, DEPENDENCY_FUNCTION_NAMES, FUNCTION_AST_KIND, FUNCTION_FIXTURE_TYPES, FUNCTION_PAYLOAD_KIND, HELPER_READS_DIALECT, - LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, MAX_FUNCTION_CALL_DEPTH, NAME_PAYLOAD_KEY, - NULL_PAYLOAD_KIND, RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY, - SET_OPERATION_PAYLOAD_KEYS, STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND, - VARIANT_FIXTURE_TYPE, WILDCARD, + LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, NAME_PAYLOAD_KEY, NULL_PAYLOAD_KIND, + RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY, SET_OPERATION_PAYLOAD_KEYS, + STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND, VARIANT_FIXTURE_TYPE, WILDCARD, }; /// Python's `bool(value)`. @@ -193,12 +193,7 @@ pub(crate) fn parse_one_or_error( sql: &str, dialect: &str, ) -> Result, String> { - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| error.to_string())?; - let options: ParseOptions = ParseOptions { - complexity_guard: Some(guard), - }; + let options: ParseOptions = polyglot_parse_options(); let parsed: Result, _> = Dialect::get_by_name(dialect) .ok_or_else(|| format!("unknown dialect {dialect}"))? .parse_with_options(sql, &options); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index 7323c2820a..c6ac5aaec3 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; -use sqlquack_syntax::{CallMode, Dialect, Expression, ParseOptions}; +use sqlquack_syntax::{CallMode, Dialect, Expression}; use crate::assembly::analysis_session::_helpers::dict_walk::{ casefold, column_name, dict_list, identifier_name, is_single_wildcard, nested, node_key, @@ -25,6 +25,7 @@ use crate::assembly::analysis_session::models::{ ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::Shapes; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::relation_markers::constants::MARKER_NAME_PREFIX; use crate::relation_markers::main::markers_as_names::markers_as_names; use crate::relation_markers::types::MarkerCall; @@ -134,7 +135,7 @@ fn parsed(sql: &str, dialect: &str) -> Result, String> { let parsed = sqlquack_parse::parse_statements_with( &named, dialect, - &ParseOptions::default(), + &sqlquack_parse_options(), CallMode::Faithful, ); let mut statements: Vec = match parsed { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index 12bab7d107..4e4ad91ae5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -14,14 +14,15 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{truthy, upper}; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::constants::{ CAST_TRANSFORM, CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, FILTER_CONTEXT, - MAX_FUNCTION_CALL_DEPTH, NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE, - SELECT_SHAPE, SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE, SELECT_SHAPE, + SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, TRANSFORM_STAR, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, WILDCARD, }; use crate::assembly::analysis_session::models::NullabilityCallback; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow, ModelRequest}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::NormalizationInput; @@ -75,12 +76,9 @@ fn analysed(input: &EnrichmentInput<'_>) -> Result { if let Some(schema) = analysis_schema(input) { options.insert("schema".to_owned(), schema); } - options.insert( - "complexityGuard".to_owned(), - json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH}), - ); - let options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options)) + let mut options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options)) .map_err(|error| format!("query analysis options: {error}"))?; + options.complexity_guard = Some(polyglot_complexity_guard()); let analysis = match analyze_query(&cleaned, options) { Ok(analysis) => analysis, Err(_) => return legacy(input, &cleaned), diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 968ee107ea..70ca4a1333 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -56,8 +56,6 @@ pub(crate) const FILTER_CONTEXT: &str = "filter"; pub(crate) const NULL_KEYWORD: &str = "NULL"; pub(crate) const RESOLVED_SOURCE_CONFIDENCE: &str = "resolved"; pub(crate) const WILDCARD: &str = "*"; -/// SQLBuild's `analyze_query` complexity guard. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; /// The dialects whose dynamic pivots the compiler proves, and those with DuckDB's PIVOT form. pub(crate) const SUPPORTED_PIVOT_DIALECTS: [&str; 3] = ["duckdb", "motherduck", "snowflake"]; pub(crate) const SIMPLIFIED_PIVOT_DIALECTS: [&str; 2] = ["duckdb", "motherduck"]; diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs index b9706b03d1..4e19530a98 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs @@ -2,12 +2,10 @@ use std::collections::HashMap; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, ParseOptions, ValidationOptions, - validate_with_dialect, -}; +use polyglot_sql::{Dialect, DialectType, ParseOptions, ValidationOptions, validate_with_dialect}; -use crate::assembly::project::constants::{INVALID_SQL, MAX_FUNCTION_CALL_DEPTH}; +use crate::assembly::project::constants::INVALID_SQL; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; @@ -40,10 +38,7 @@ pub(crate) fn syntax_error( }; let dialect_type: DialectType = parser_dialect(Some(dialect)) .map_err(|unknown| SyntaxFailure::UnknownDialect(unknown.to_owned()))?; - let complexity_guard = ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..Default::default() - }; + let complexity_guard = polyglot_complexity_guard(); let polyglot: Dialect = Dialect::get(dialect_type); Ok(match mode { SyntaxMode::Validate => { diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index 91b7a61a5e..6d63fc9a7c 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -16,7 +16,6 @@ pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const PYTHON_NONE: &str = "None"; pub(crate) const PYTHON_TRUE: &str = "True"; pub(crate) const PYTHON_FALSE: &str = "False"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; diff --git a/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs b/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs index 2282e89ede..1d001868bb 100644 --- a/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs +++ b/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs @@ -1,5 +1,6 @@ //! Resolve every reference to one relation column, with the spans a rename rewrites. +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use polyglot_sql::{ Dialect, DialectType, Expression, MappingSchema, Resolver, Scope, ValidationSchema, build_scope, mapping_schema_from_validation_schema_with_dialect, walk_in_scope, @@ -109,7 +110,9 @@ pub(crate) fn analyze_json(request_json: &str) -> Result { request.dialect ) })?; - let Ok(statements) = Dialect::get(dialect).parse(&request.sql) else { + let Ok(statements) = + Dialect::get(dialect).parse_with_options(&request.sql, &polyglot_parse_options()) + else { return serde_json::to_string(&ReferenceResponse::default()) .map_err(|error| error.to_string()); }; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs index 330f9618fa..3296318703 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs @@ -2,8 +2,8 @@ use polyglot_sql::Dialect; -use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options; use crate::compiler::constants::FUNCTION_TYPE_DIALECTS; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// The error for a type the adapter's dialect cannot parse; adapters without a dialect pass. pub(crate) fn function_type_error( @@ -17,9 +17,8 @@ pub(crate) fn function_type_error( else { return Ok(None); }; - let options = guarded_parse_options().map_err(|error| error.to_string())?; Ok(Dialect::get(*dialect) - .parse_data_type_with_options(type_sql, &options) + .parse_data_type_with_options(type_sql, &polyglot_parse_options()) .err() .map(|error| { format!( diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs index bfe54fc9de..e29031ff03 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs @@ -1,2 +1 @@ pub(crate) mod function_types; -pub(crate) mod parse_options; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs deleted file mode 100644 index 773f7acad5..0000000000 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The Polyglot parse options SQLBuild's wheel proxy applies to every parse. - -use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; - -use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; - -/// The proxy's complexity guard, decoded as the wheel decodes it. -pub(crate) fn guarded_parse_options() -> Result { - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - }))?; - Ok(ParseOptions { - complexity_guard: Some(guard), - }) -} diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs index adaf9e1f93..5a021d54c4 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs @@ -5,6 +5,7 @@ use polyglot_sql::tokens::{Token, TokenType}; use polyglot_sql::{Dialect, DialectType, Expression}; use crate::contracts::models::LiteralLocation; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// How deep the walk follows set operations and passthrough columns before it stops. const MAX_DEPTH: usize = 64; @@ -69,7 +70,9 @@ pub(crate) fn parsed_query(sql: &str, dialect: &str) -> Option { let Ok(dialect) = dialect.parse::() else { return None; }; - let Ok(mut statements) = polyglot_sql::parse(sql, dialect) else { + let Ok(mut statements) = + polyglot_sql::parse_with_options(sql, dialect, &polyglot_parse_options()) + else { return None; }; (statements.len() == 1).then(|| statements.pop()).flatten() diff --git a/crates/sqlbuild-analysis/src/diff_selection/constants.rs b/crates/sqlbuild-analysis/src/diff_selection/constants.rs index 13daec5375..ad63030eee 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/constants.rs @@ -13,4 +13,3 @@ pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; pub(crate) const RAW_QUERY_READ_ONLY_ROOTS: [&str; 5] = ["except", "intersect", "select", "union", "values"]; -pub(crate) const RAW_QUERY_GUARD: &str = r#"{"maxFunctionCallDepth": 512}"#; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs index c553e213e1..6e879e1e36 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -1,25 +1,19 @@ //! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. +use polyglot_sql::Expression; use polyglot_sql::dialects::Dialect; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; -use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; +use crate::diff_selection::constants::RAW_QUERY_READ_ONLY_ROOTS; use crate::diff_selection::models::RawQueryShape; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// The query's shape, or the text of the exception the wheel raises for it. pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; - let guard: ComplexityGuardOptions = - serde_json::from_str(RAW_QUERY_GUARD).map_err(|error| error.to_string())?; let mut statements: Vec = parser - .parse_with_options( - sql, - &ParseOptions { - complexity_guard: Some(guard), - }, - ) + .parse_with_options(sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; if statements.len() != 1 { return Err(format!("Expected 1 statement, found {}", statements.len())); diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 0f0ca871f0..14f19e6e7b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -26,6 +26,7 @@ pub mod lineage_graph; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; +pub mod parse_guard; pub mod planner_changes; pub mod planner_fixtures; pub mod planner_identity; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs index 93f982d537..eacc193e3f 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, Expression, ParseOptions}; +use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; use crate::lineage::_helpers::projections::{ UnreadableExpression, classify_transform, is_star, projection_output_name, @@ -10,29 +10,19 @@ use crate::lineage::_helpers::projections::{ }; use crate::lineage::_helpers::references::{PhysicalResource, normalized_sql, physical_resources}; use crate::lineage::_helpers::stars::star_lineage; -use crate::lineage::constants::{KIND_SELECT, KIND_UNION, MAX_FUNCTION_CALL_DEPTH}; +use crate::lineage::constants::{KIND_SELECT, KIND_UNION}; use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSource, LineageTransformKind, }; -/// The parse options SQLBuild's Polyglot proxy gives `parse_one`, decoded as the wheel does. -pub(crate) fn proxy_parse_options() -> Result { - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - }))?; - Ok(ParseOptions { - complexity_guard: Some(guard), - }) -} - pub(crate) struct ParsedModel<'a> { pub(crate) query_sql: &'a str, pub(crate) inferred_names: &'a [String], pub(crate) schema: &'a HashMap>, /// The parser dialect, or the dialect name Polyglot does not know. pub(crate) dialect: Result, - pub(crate) options: &'a Result, + pub(crate) options: &'a ParseOptions, } /// The model's lineage outcome; fails on a tree the lineage walk cannot read. @@ -41,10 +31,7 @@ pub(crate) fn parsed_model_lineage(model: &ParsedModel<'_>) -> Result dialect, Err(name) => return Ok(FastLineageOutcome::UnknownDialect(name.to_owned())), }; - let options: &ParseOptions = model - .options - .as_ref() - .map_err(|_| "the parse options do not build".to_owned())?; + let options: &ParseOptions = model.options; let physical = physical_resources(model.query_sql); let sql = normalized_sql(model.query_sql); let mut statements = match Dialect::get(dialect).parse_with_options(&sql, options) { diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs index 302db0fdb6..8ec0387fe7 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs @@ -15,12 +15,13 @@ use crate::lineage::_helpers::references::{ PhysicalResource, normalized_sql, physical_resource_name, physical_resources, }; use crate::lineage::_helpers::stars::star_lineage; -use crate::lineage::constants::{MAX_FUNCTION_CALL_DEPTH, STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE}; +use crate::lineage::constants::{STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE}; use crate::lineage::models::{ LineageColumn, LineageConfidence, LineageNullability, LineageResourceType, LineageSchemaResource, LineageSource, LineageTransformKind, RichLineageColumn, RichLineageOutcome, RichSchemaResource, }; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; /// What every model of one request shares: the dialect, the guard and the known tables. pub(crate) struct RichContext { @@ -42,10 +43,7 @@ impl RichContext { ) -> Self { Self { dialect, - guard: ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..ComplexityGuardOptions::default() - }, + guard: polyglot_complexity_guard(), tables: schema_tables(schema, referenced), names, } diff --git a/crates/sqlbuild-analysis/src/lineage/constants.rs b/crates/sqlbuild-analysis/src/lineage/constants.rs index cf37260374..af4ceac67e 100644 --- a/crates/sqlbuild-analysis/src/lineage/constants.rs +++ b/crates/sqlbuild-analysis/src/lineage/constants.rs @@ -21,8 +21,6 @@ pub(crate) const PAYLOAD_COLUMN: &str = "column"; pub(crate) const PAYLOAD_NAME: &str = "name"; pub(crate) const PAYLOAD_TABLE: &str = "table"; pub(crate) const SET_OPERATION_SIDES: [&str; 2] = ["left", "right"]; -/// The function-call depth SQLBuild's Polyglot proxy passes on every parse. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const REFERENCE_PREFIX: &str = "__"; pub(crate) const UDF_CALL: &str = "__udf("; pub(crate) const PHYSICAL_RESOURCE_PREFIX: &str = "__sqlbuild_"; diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index d20bf68d63..5e1145f5be 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -11,13 +11,12 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::main::fast_lineage::fast_lineage; use crate::analysis_route::models::AnalysisRoute; -use crate::lineage::_helpers::parsed_lineage::{ - ParsedModel, parsed_model_lineage, proxy_parse_options, -}; +use crate::lineage::_helpers::parsed_lineage::{ParsedModel, parsed_model_lineage}; use crate::lineage::_helpers::references::physical_resources; use crate::lineage::_helpers::stars::{schema_mapping, star_lineage}; use crate::lineage::main::parser_dialect::parser_dialect; use crate::lineage::models::{FastLineageModel, FastLineageOutcome, FastLineageRequest}; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::main::new_analysis_pool::new_analysis_pool; use crate::semantic_validation::models::ProjectCatalog; @@ -56,7 +55,7 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec = proxy_parse_options(); + let options: ParseOptions = polyglot_parse_options(); request .models .par_iter() diff --git a/crates/sqlbuild-analysis/src/parse_guard/constants.rs b/crates/sqlbuild-analysis/src/parse_guard/constants.rs new file mode 100644 index 0000000000..1d65848990 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/constants.rs @@ -0,0 +1,4 @@ +//! Constants of parse_guard. + +/// The deepest function-call nesting a Polyglot or SQLQuack shared or Snowflake parse accepts. +pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs new file mode 100644 index 0000000000..793cfb6cd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs @@ -0,0 +1,3 @@ +pub mod polyglot_complexity_guard; +pub mod polyglot_parse_options; +pub mod sqlquack_parse_options; diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs new file mode 100644 index 0000000000..1114b6627d --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs @@ -0,0 +1,14 @@ +//! `polyglot_complexity_guard`: Polyglot's default limits with SQLBuild's function-call depth. + +use polyglot_sql::ComplexityGuardOptions; + +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; + +/// Polyglot's default complexity limits, allowing SQLBuild's function-call depth. +#[must_use] +pub fn polyglot_complexity_guard() -> ComplexityGuardOptions { + ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs new file mode 100644 index 0000000000..05e0e6760d --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs @@ -0,0 +1,13 @@ +//! `polyglot_parse_options`: the options every SQLBuild Polyglot parse passes. + +use polyglot_sql::ParseOptions; + +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; + +/// Polyglot parse options under SQLBuild's complexity guard. +#[must_use] +pub fn polyglot_parse_options() -> ParseOptions { + ParseOptions { + complexity_guard: Some(polyglot_complexity_guard()), + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs b/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs new file mode 100644 index 0000000000..7feafefee6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs @@ -0,0 +1,16 @@ +//! `sqlquack_parse_options`: the options every SQLBuild SQLQuack parse passes. + +use sqlquack_syntax::{ComplexityGuardOptions, ParseOptions}; + +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; + +/// SQLQuack's default complexity limits, allowing SQLBuild's function-call depth. +#[must_use] +pub fn sqlquack_parse_options() -> ParseOptions { + ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + }), + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/mod.rs new file mode 100644 index 0000000000..c636c77864 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/mod.rs @@ -0,0 +1,7 @@ +//! One call depth for Polyglot and SQLQuack's shared and Snowflake parsers, not per-dialect ones. + +pub mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests.rs b/crates/sqlbuild-analysis/src/parse_guard/tests.rs new file mode 100644 index 0000000000..6d2a051968 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_call_depth.rs"] +mod test_call_depth; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs new file mode 100644 index 0000000000..13be5c2772 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs @@ -0,0 +1,70 @@ +use polyglot_sql::DialectType; +use sqlquack_syntax::{CallMode, Dialect}; + +use crate::analysis_route::models::SnowflakeDialect; +use crate::analysis_route::types::NativeDialect; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; + +/// A query selecting `depth` nested `REPLACE` calls around one column. +pub(super) fn nested_calls(depth: usize) -> String { + let closings: String = (0..depth) + .map(|level| format!(", 'a{level}', 'b')")) + .collect(); + format!( + "SELECT {}name{closings} AS cleaned FROM races", + "REPLACE(".repeat(depth) + ) +} + +/// `work`'s result on a thread with Rust's default 2 MiB stack, as rayon and `std::thread` use. +pub(super) fn on_default_stack(work: impl FnOnce() -> T + Send) -> T { + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(2 * 1024 * 1024) + .spawn_scoped(scope, work) + .expect("thread starts") + .join() + .expect("parse does not overflow the stack") + }) +} + +/// Each parser SQLBuild guards, with whether it accepts `sql` or the error it raised. +pub(super) fn parses(sql: &str) -> Vec<(&'static str, Result<(), String>)> { + vec![ + ( + "native Snowflake module", + SnowflakeDialect + .parse(sql) + .map(|_| ()) + .map_err(|gap| gap.message), + ), + ( + "SQLQuack Snowflake", + sqlquack_parse::parse_statements_with( + sql, + Dialect::Snowflake, + &sqlquack_parse_options(), + CallMode::Faithful, + ) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ( + "Polyglot Snowflake", + polyglot_sql::parse_with_options( + sql, + DialectType::Snowflake, + &polyglot_parse_options(), + ) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ( + "Polyglot DuckDB", + polyglot_sql::parse_with_options(sql, DialectType::DuckDB, &polyglot_parse_options()) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ] +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs new file mode 100644 index 0000000000..f9d812847c --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs @@ -0,0 +1,55 @@ +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::parse_guard::tests::helpers::{nested_calls, on_default_stack, parses}; +use crate::parse_guard::tests::test_types::CallDepthTestCase; + +#[test] +fn given_nested_calls_when_parsing_on_a_default_stack_then_each_parser_applies_the_shared_limit() { + let test_cases = [ + CallDepthTestCase { + description: "one past the libraries' default of 64", + depth: 65, + expected_error: None, + }, + CallDepthTestCase { + description: "a hundred deep", + depth: 100, + expected_error: None, + }, + CallDepthTestCase { + description: "exactly the limit", + depth: MAX_FUNCTION_CALL_DEPTH, + expected_error: None, + }, + CallDepthTestCase { + description: "one past the limit names it", + depth: MAX_FUNCTION_CALL_DEPTH + 1, + expected_error: Some(format!( + "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value {} exceeds configured limit \ + {MAX_FUNCTION_CALL_DEPTH}", + MAX_FUNCTION_CALL_DEPTH + 1 + )), + }, + ]; + + for test_case in test_cases { + let sql: String = nested_calls(test_case.depth); + let expected_length: usize = test_case.expected_error.as_ref().map_or(0, String::len); + + for (parser, outcome) in on_default_stack(|| parses(&sql)) { + let guard_error: Option = outcome.err().map(|error| { + error + .get(error.find("E_GUARD").unwrap_or(0)..) + .unwrap_or_default() + .chars() + .take(expected_length) + .collect() + }); + + assert_eq!( + guard_error, test_case.expected_error, + "{}: {parser}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs new file mode 100644 index 0000000000..ab236298be --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct CallDepthTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + /// The guard error every parser raises, from its code, or `None` when each accepts the SQL. + pub(super) expected_error: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs index 5afda5f19d..22f36ac948 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -8,6 +8,7 @@ use polyglot_sql::{ }; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; use crate::planner_fixtures::models::FixtureStarRelation; @@ -48,7 +49,7 @@ pub fn star_keys( tables, strict: None, }), - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), }; let Ok(sql) = stub_markers(sql) else { return Vec::new(); diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs index a84b3ee534..f6c6792626 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs @@ -1,20 +1,15 @@ //! Parse migration SQL under the compiler's complexity limit. -use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; use polyglot_sql::dialects::Dialect; -use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; -use serde_json::{Value, json}; + +use serde_json::Value; + +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { let dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("unknown dialect {dialect}"))?; - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| error.to_string())?; - let options = ParseOptions { - complexity_guard: Some(guard), - }; - let Ok(mut statements) = dialect.parse_with_options(sql, &options) else { + let Ok(mut statements) = dialect.parse_with_options(sql, &polyglot_parse_options()) else { return Ok(None); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index cde00841a5..6bd5144f1e 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -15,6 +15,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; use crate::constants::{UNKNOWN_SQL_TYPE, VARCHAR_SQL_TYPE}; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use sqlbuild_core::constants::SQL_WILDCARD; const DEFAULT_WORKERS: usize = 4; @@ -89,6 +91,7 @@ impl AnalysisRequest { check_references: true, strict: Some(true), strict_syntax: false, + complexity_guard: Some(polyglot_complexity_guard()), ..Default::default() } } @@ -612,16 +615,9 @@ fn try_borrowed_query( if !matches!(dialect, DialectType::Snowflake | DialectType::DuckDB) { return Err(Box::new(work)); } - let Ok(mut statements) = polyglot_sql::parse_with_options( - &work.query.sql, - dialect, - &polyglot_sql::ParseOptions { - complexity_guard: Some(polyglot_sql::ComplexityGuardOptions { - max_function_call_depth: Some(128), - ..Default::default() - }), - }, - ) else { + let Ok(mut statements) = + polyglot_sql::parse_with_options(&work.query.sql, dialect, &polyglot_parse_options()) + else { return Err(Box::new(work)); }; if statements.len() != 1 { @@ -1036,7 +1032,7 @@ fn compile_query(mut request: AnalysisRequest, project_projections: bool) -> Com let compiled = polyglot_sql::compile_required_query_analysis( &request.sql, AnalyzeQueryOptions { - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), dialect, schema: request.schema, }, @@ -1105,7 +1101,7 @@ fn query_analysis( } }; let options = AnalyzeQueryOptions { - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), dialect, schema: request.schema, }; diff --git a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs index 28fe14df79..ecd6d23d29 100644 --- a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs @@ -1,12 +1,13 @@ //! Python's dict walk over each parsed statement: tables, CTE names and temporary tables. -use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; -use serde_json::{Map, Value, json}; +use polyglot_sql::{Dialect, Expression}; +use serde_json::{Map, Value}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::relation_names::constants::{ - CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, - MAX_FUNCTION_CALL_DEPTH, NAME_KEY, QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, + CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, NAME_KEY, + QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, }; use crate::relation_names::errors::RelationNamesFailure; use crate::relation_names::models::{ExtractedRelations, RelationName}; @@ -53,13 +54,7 @@ fn parsed_statements( let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| { RelationNamesFailure::UnknownDialect(format!("Unknown dialect: {dialect}")) })?; - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| RelationNamesFailure::Internal(error.to_string()))?; - let options: ParseOptions = ParseOptions { - complexity_guard: Some(guard), - }; - match parser.parse_with_options(sql, &options) { + match parser.parse_with_options(sql, &polyglot_parse_options()) { Ok(statements) => Ok(Some(statements)), Err(_polyglot_error) => Ok(None), } diff --git a/crates/sqlbuild-analysis/src/relation_names/constants.rs b/crates/sqlbuild-analysis/src/relation_names/constants.rs index f8cab78a94..1f93241a40 100644 --- a/crates/sqlbuild-analysis/src/relation_names/constants.rs +++ b/crates/sqlbuild-analysis/src/relation_names/constants.rs @@ -11,5 +11,3 @@ pub(crate) const CREATE_TABLE_KEY: &str = "create_table"; pub(crate) const TEMPORARY_KEY: &str = "temporary"; pub(crate) const QUALIFIER_SEPARATOR: char = '.'; pub(crate) const GENERIC_DIALECT: &str = "generic"; -/// The guarded proxy's `maxFunctionCallDepth` for the wheel's `parse`. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; diff --git a/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs index f09b0f5f66..5931fd33ff 100644 --- a/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs +++ b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs @@ -1,9 +1,10 @@ use std::collections::HashMap; use std::rc::Rc; -use polyglot_sql::{Dialect, Expression, ParseOptions}; +use polyglot_sql::{Dialect, Expression}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::required_columns::_helpers::queries::query_reads; use crate::required_columns::_helpers::relations::{RelationMap, RelationReads, unique_reads}; use crate::required_columns::constants::{GENERIC_DIALECT, TABLE_FUNCTION_STUB_PREFIX}; @@ -28,7 +29,7 @@ pub fn resolve_required_columns( .map_err(RequiredColumnsError::Normalization)?; let parser: Dialect = Dialect::get_by_name(dialect_name) .ok_or_else(|| RequiredColumnsError::UnknownDialect(dialect_name.to_owned()))?; - let Ok(mut statements) = parser.parse_with_options(&sql, &ParseOptions::default()) else { + let Ok(mut statements) = parser.parse_with_options(&sql, &polyglot_parse_options()) else { return Ok(Vec::new()); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs index ca3f08dca7..9f240f0365 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs @@ -3,13 +3,11 @@ use std::collections::{HashMap, HashSet}; use polyglot_sql::tokens::TokenType; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, ParseOptions, -}; +use polyglot_sql::{Dialect, DialectType, Expression, ExpressionWalk}; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_checks::constants::{ - CLOSE_PARENTHESIS, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, OPEN_PARENTHESIS, - PROJECTION_SEPARATOR, + CLOSE_PARENTHESIS, GENERIC_DIALECT, OPEN_PARENTHESIS, PROJECTION_SEPARATOR, }; use crate::semantic_checks::models::SemanticFailure; use crate::semantic_validation::main::normalize::normalize_analysis_sql; @@ -120,14 +118,7 @@ pub(crate) fn parsed_model( ) -> Result, SemanticFailure> { let sql = normalized_sql(query_sql, dialect)?; let parser = polyglot_dialect(dialect.unwrap_or(GENERIC_DIALECT))?; - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - })) - .map_err(|_| SemanticFailure::internal("the parse guard options do not build"))?; - let options = ParseOptions { - complexity_guard: Some(guard), - }; - let Ok(mut statements) = parser.parse_with_options(&sql, &options) else { + let Ok(mut statements) = parser.parse_with_options(&sql, &polyglot_parse_options()) else { return Ok(None); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 98800e3e4d..0b0b44e949 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -5,7 +5,6 @@ pub(crate) const COMPARISON_CODE: &str = "B217"; pub(crate) const SQL_TEST_COLUMN_CODE: &str = "B302"; pub(crate) const SEMANTIC_CODE_PREFIX: &str = "B"; pub(crate) const MODEL_RESOURCE_TYPE: &str = "model"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: u64 = 512; pub(crate) const GENERIC_DIALECT: &str = "generic"; pub(crate) const OPEN_PARENTHESIS: &str = "("; pub(crate) const CLOSE_PARENTHESIS: &str = ")"; diff --git a/crates/sqlbuild-analysis/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs index beeedb31e0..623070bd77 100644 --- a/crates/sqlbuild-analysis/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -1,5 +1,7 @@ //! On-demand direct non-projection column usage extraction. +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, Resolver, SchemaValidationOptions, ValidationSchema, build_scope, mapping_schema_from_validation_schema_with_dialect, @@ -62,6 +64,7 @@ pub fn analyze_json(request_json: &str) -> Result { &SchemaValidationOptions { check_references: true, strict: Some(true), + complexity_guard: Some(polyglot_complexity_guard()), ..SchemaValidationOptions::default() }, ); @@ -70,7 +73,7 @@ pub fn analyze_json(request_json: &str) -> Result { .map_err(|error| error.to_string()); } let statements = Dialect::get(dialect) - .parse(&request.sql) + .parse_with_options(&request.sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; let schema = mapping_schema_from_validation_schema_with_dialect(&request.schema, dialect); let mut uses: Vec = Vec::new(); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index 4a5dd340db..5262ddcd16 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -1,5 +1,7 @@ //! Compile-owned schema catalog and native, batched binding requests. +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::main::normalize_batch; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -68,7 +70,7 @@ impl ProjectCatalog { ) -> Result)>, String> { use polyglot_sql::{Expression, ExpressionWalk}; let parsed = Dialect::get(self.dialect) - .parse(sql) + .parse_with_options(sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; let mut identifiers: HashMap = HashMap::new(); for expression in &parsed { @@ -151,10 +153,7 @@ impl ProjectCatalog { check_references: true, semantic: true, strict: Some(true), - complexity_guard: Some(polyglot_sql::ComplexityGuardOptions { - max_function_call_depth: Some(512), - ..Default::default() - }), + complexity_guard: Some(polyglot_complexity_guard()), ..Default::default() }, tables: HashMap::new(), diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index a60d49f7ab..4778cb130a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -1,5 +1,6 @@ //! SQLBuild codes and dialect-specific diagnostic evidence. +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; use crate::semantic_validation::_helpers::sqlite_temporal_casts; @@ -76,8 +77,9 @@ struct DiagnosticContext<'a> { impl DiagnosticContext<'_> { fn parsed(&mut self) -> &polyglot_sql::Result> { - self.statements - .get_or_insert_with(|| Dialect::get(self.dialect).parse(self.sql)) + self.statements.get_or_insert_with(|| { + Dialect::get(self.dialect).parse_with_options(self.sql, &polyglot_parse_options()) + }) } fn map(&mut self, mut error: ValidationError) -> Result, String> { diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index fb9ca0d11e..5e0c5393df 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -14,6 +14,8 @@ pub mod normalize_native_renamed; pub mod sqlbuild_code; pub mod text_findings; pub mod truth_test_spans; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::models::ProjectCatalog; use polyglot_sql::{ @@ -88,6 +90,13 @@ pub(super) fn validation_result(request: ValidationRequest) -> Result bool { @@ -18,11 +19,8 @@ pub(crate) fn python_space(character: char) -> bool { /// Whether a CREATE statement's text parses as a definition that needs no warehouse. pub(crate) fn metadata_only_create(query_text: &str) -> bool { - let options: ParseOptions = ParseOptions { - complexity_guard: None, - }; - let Ok(mut statements) = - Dialect::get(DialectType::Snowflake).parse_with_options(query_text, &options) + let Ok(mut statements) = Dialect::get(DialectType::Snowflake) + .parse_with_options(query_text, &polyglot_parse_options()) else { return false; }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index 1dc7f7981e..4af625b2e9 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_call_depth.rs"] +mod test_call_depth; #[path = "tests/test_definitions.rs"] mod test_definitions; #[path = "tests/test_expansion.rs"] diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index 105be1d1f6..5ccf28eae7 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -233,3 +233,18 @@ pub(super) fn raw_prefix_rules() -> sqlbuild_sqltext::sql_scan::models::LexicalS ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() } } + +/// An expression macro whose body nests `depth` `REPLACE` calls around its argument. +pub(super) fn nested_replace_macro(depth: usize) -> String { + let mut source: String = String::from( + "MACRO (description \"Cleaned name\", arguments (column_name VARCHAR), returns VARCHAR);\n\n", + ); + let closings: String = (0..depth) + .map(|level| format!(", 'a{level}', 'b')")) + .collect(); + source.push_str(&format!( + "{}@column_name{closings}\n", + "REPLACE(".repeat(depth) + )); + source +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs new file mode 100644 index 0000000000..0df580118c --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs @@ -0,0 +1,45 @@ +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::sql_macros::tests::helpers::{expanded_for, nested_replace_macro}; +use crate::sql_macros::tests::test_types::NestedMacroTestCase; + +#[test] +fn given_a_deeply_nested_macro_body_when_expanding_for_snowflake_then_the_shared_limit_applies() { + let test_cases = [ + NestedMacroTestCase { + description: "a hundred nested calls expand", + depth: 100, + expected_outcome: Ok(100), + }, + NestedMacroTestCase { + description: "one past the limit is P015 naming the limit", + depth: MAX_FUNCTION_CALL_DEPTH + 1, + expected_outcome: Err(format!( + "body of macro cleaned_name() is not a valid expression macro: \ + E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value {} exceeds configured limit \ + {MAX_FUNCTION_CALL_DEPTH}", + MAX_FUNCTION_CALL_DEPTH + 1 + )), + }, + ]; + + for test_case in test_cases { + let source: String = nested_replace_macro(test_case.depth); + + let outcome = expanded_for( + "snowflake", + &[("cleaned_name", &source)], + "SELECT @sql.cleaned_name(race_name) AS cleaned FROM races", + ) + .map(|sql| sql.matches("REPLACE(").count()) + .map_err(|error| format!("{}: {}", error.code, error.message)); + + assert_eq!( + outcome, + test_case + .expected_outcome + .map_err(|message| format!("P015: {message}")), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 26fdebc00a..a8a3668922 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -135,3 +135,10 @@ pub(crate) struct ProbeSchemaLookupTestCase { pub(crate) relation: &'static str, pub(crate) expected_column: Option<&'static str>, } + +pub(super) struct NestedMacroTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + /// The expansion's `REPLACE` count, or the P015 message text when the body is too deep. + pub(super) expected_outcome: Result, +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs index 705b3a0834..f9adb36c2e 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs @@ -1,7 +1,8 @@ //! Parse SQL into statements with node spans. -use sqlquack_syntax::{CallMode, Expression, ParseOptions}; +use sqlquack_syntax::{CallMode, Expression}; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::sql_tree::constants::GUARD_CODE_PREFIX; use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::TreeParser; @@ -12,7 +13,7 @@ pub(crate) fn parse(sql: &str, parser: TreeParser) -> Result, Sq TreeParser::Shared(dialect) => sqlquack_parse::parse_statements_with( sql, dialect, - &ParseOptions::default(), + &sqlquack_parse_options(), CallMode::FaithfulLocated, ) .map_err(|error| shared_error(&error.to_string())), diff --git a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs index b709dc1b9f..fe8e746a20 100644 --- a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs +++ b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs @@ -2,9 +2,10 @@ use serde_json::Value; use sqlquack_parse::parse_strict; +use sqlquack_syntax::Expression; use sqlquack_syntax::tokens::models::Dialect; -use sqlquack_syntax::{Expression, ParseOptions}; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::warehouse_types::constants::{CAST_KEY, CAST_TARGET_KEY}; /// `spelling` read by `sqlquack-syntax-postgres`'s type grammar. @@ -17,7 +18,7 @@ pub(crate) fn postgres_type(spelling: &str) -> Result { /// `spelling` read as a cast target by SQLQuack's core parser for `dialect`. pub(crate) fn core_type(spelling: &str, dialect: Dialect) -> Result { let statements: Vec = - parse_strict(&cast(spelling), dialect, &ParseOptions::default()) + parse_strict(&cast(spelling), dialect, &sqlquack_parse_options()) .map_err(|error| format!("{error:?}"))?; single_cast_target(spelling, &statements) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 510cd06cc0..108c373165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -8,6 +8,7 @@ use sqlbuild_analysis::fingerprint_storage::constants::{ COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; +use sqlbuild_analysis::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; use sqlbuild_cli::compute_logs::constants::{ COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, @@ -141,6 +142,7 @@ pub(crate) const EXPORTED_INTEGERS: &[(&str, i64)] = &[ "CUSTOM_HOST_PROTOCOL_VERSION", CUSTOM_HOST_PROTOCOL_VERSION as i64, ), + ("MAX_FUNCTION_CALL_DEPTH", MAX_FUNCTION_CALL_DEPTH as i64), ]; /// Text tuples exported under their Python names. pub(crate) const EXPORTED_TEXT_TUPLES: &[(&str, &[&str])] = &[( diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index 02fcf32612..44c326018b 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::LazyLock; use regex::Regex; +use sqlbuild_analysis::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_core::text::main::python_isspace::python_isspace; @@ -21,8 +22,8 @@ use crate::lint_text::constants::{ BLOCK_COMMENT_OPENER, DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, EXPRESSION_SELECT_PREFIX, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, GLOBAL_NAMED_ROOTS, GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, - MACRO_HEADER_KIND, MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, - OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, + MACRO_HEADER_KIND, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, OMITTED_CEREMONIAL_SELECT_SQL, + OMITTED_SELECT_SUFFIX_PATTERN, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index e2dac213d3..a5188d2bf4 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -67,7 +67,6 @@ pub(crate) const UNIT_TEST_ROOT: [&str; 2] = ["tests", "unit"]; pub(crate) const SQL_SUFFIX: &str = ".sql"; pub(crate) const FORMAT_API_VERSION: u32 = 1; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const OMITTED_CEREMONIAL_SELECT_SQL: &str = "\nSELECT 1"; pub(crate) const OMITTED_SELECT_SUFFIX_PATTERN: &str = r"(?i)\s*\bSELECT\s+1\s*\z"; pub(crate) const FIXTURE_HEADER_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index 46ca4fa8af..ca5e1ce889 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -2,20 +2,18 @@ use crate::query_facts::_helpers::flatten::{flatten_call, flatten_input, whole_v use crate::query_facts::_helpers::shape::stub_call; use crate::query_facts::_helpers::tokens::{ComparisonOperator, QueryTokens, SelectTokens}; use crate::query_facts::constants::{ - MAX_AST_DEPTH, MAX_FUNCTION_CALL_DEPTH, SNOWFLAKE_DATE_PART_ARGUMENTS, TABLE_FUNCTION_STUB, - TRIPLE_QUOTE_LENGTH, + MAX_AST_DEPTH, SNOWFLAKE_DATE_PART_ARGUMENTS, TABLE_FUNCTION_STUB, TRIPLE_QUOTE_LENGTH, }; use crate::query_facts::models::{ CallKind, ColumnFact, ComparisonFacts, ComparisonSpan, CteFacts, ExpressionItem, ItemFacts, ModelShape, ParsedFixture, QueryFacts, RelationFacts, SelectFacts, SetFacts, SourceText, }; +use sqlbuild_analysis::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use sqlbuild_analysis::sql_tree::main::deeper_than::deeper_than; use sqlbuild_analysis::sql_tree::main::shared_dialect::shared_dialect; use sqlquack_syntax::expressions::names::models::{Identifier, Literal, TableRef}; use sqlquack_syntax::expressions::queries::models::{Select, With}; -use sqlquack_syntax::{ - CallMode, ComplexityGuardOptions, Dialect, Expression, ExpressionWalk, ParseOptions, -}; +use sqlquack_syntax::{CallMode, Dialect, Expression, ExpressionWalk}; use std::cell::{Cell, RefCell}; use std::collections::{HashMap, HashSet}; use std::ops::Range; @@ -23,12 +21,7 @@ use std::ops::Range; fn parse_query(sql: &str, dialect_name: &str) -> Result<(Expression, Dialect), String> { let dialect = shared_dialect(dialect_name) .ok_or_else(|| format!("no SQL parser for dialect {dialect_name}"))?; - let options = ParseOptions { - complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..Default::default() - }), - }; + let options = sqlquack_parse_options(); let mut statements = sqlquack_parse::parse_statements_with(sql, dialect, &options, CallMode::FaithfulLocated) .map_err(|error| error.to_string())?; diff --git a/crates/sqlbuild-rules/src/query_facts/constants.rs b/crates/sqlbuild-rules/src/query_facts/constants.rs index ae3601132a..df202a3229 100644 --- a/crates/sqlbuild-rules/src/query_facts/constants.rs +++ b/crates/sqlbuild-rules/src/query_facts/constants.rs @@ -5,7 +5,6 @@ pub(crate) const TABLE_FUNCTION_STUB: &str = "__sqlbuild_table_function_"; pub(crate) const UDF_STUB: &str = "__sqlbuild_udf_"; pub(crate) const PLACEHOLDER_PREFIX: &[u8] = b"@@@"; pub(crate) const PLACEHOLDER_VALUE: &str = "NULL"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 128; /// The deepest statement built-in rules read, Polyglot's former AST-depth limit. pub(crate) const MAX_AST_DEPTH: usize = 512; pub(crate) const UDF_CALL: &str = "__udf"; diff --git a/scripts/check_parse_guard.py b/scripts/check_parse_guard.py new file mode 100644 index 0000000000..d4cfa4af23 --- /dev/null +++ b/scripts/check_parse_guard.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the shared parse guard check.""" + +from __future__ import annotations + +from scripts.parse_guard.main.check import run_parse_guard_check + + +def main(argv: list[str] | None = None) -> int: + """Check that Polyglot and SQLQuack's shared and Snowflake parses use the shared guard.""" + return run_parse_guard_check(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/native_layering/main/check.py b/scripts/native_layering/main/check.py index 4c4e6a2711..1db7c6318c 100644 --- a/scripts/native_layering/main/check.py +++ b/scripts/native_layering/main/check.py @@ -2,26 +2,22 @@ from __future__ import annotations -import argparse -import sys from pathlib import Path from scripts.native_layering._helpers.layering import get_native_layering_errors from scripts.native_layering._helpers.versions import get_native_version_errors +from scripts.root_check.main.run_root_check import run_root_check def run_native_layering_check(argv: list[str] | None = None) -> int: """Fail when a native crate breaks the declared layering or the release versions differ.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) - errors: tuple[str, ...] = get_native_layering_errors( - arguments.root - ) + get_native_version_errors(arguments.root) - if errors: - print("Native crate layering is invalid:", file=sys.stderr) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Native crate layering is valid.") - return 0 + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_layering_and_version_errors, + messages=("Native crate layering is invalid:", "Native crate layering is valid."), + ) + + +def _layering_and_version_errors(root: Path) -> tuple[str, ...]: + return get_native_layering_errors(root) + get_native_version_errors(root) diff --git a/scripts/native_stub/main/check.py b/scripts/native_stub/main/check.py index 17af5ff84c..3d14757250 100644 --- a/scripts/native_stub/main/check.py +++ b/scripts/native_stub/main/check.py @@ -2,27 +2,28 @@ from __future__ import annotations -import argparse import importlib -import sys from pathlib import Path from scripts.native_stub._helpers.declarations import stub_drift from scripts.native_stub.constants import NATIVE_MODULE, STUB_PATH +from scripts.root_check.main.run_root_check import run_root_check def run_native_stub_check(argv: list[str] | None = None) -> int: """Fail when the stub declares a runtime name the built native extension does not register.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_unregistered_names, + messages=( + f"{STUB_PATH} declares names {NATIVE_MODULE} does not register:", + f"{STUB_PATH} declares only registered names.", + ), + ) + + +def _unregistered_names(root: Path) -> tuple[str, ...]: registered: frozenset[str] = frozenset(dir(importlib.import_module(NATIVE_MODULE))) - stub_text: str = (arguments.root / STUB_PATH).read_text(encoding="utf-8") - missing: tuple[str, ...] = stub_drift(stub_text=stub_text, registered=registered) - if missing: - print(f"{STUB_PATH} declares names {NATIVE_MODULE} does not register:", file=sys.stderr) - for name in missing: - print(f"- {name}", file=sys.stderr) - return 1 - print(f"{STUB_PATH} declares only registered names.") - return 0 + stub_text: str = (root / STUB_PATH).read_text(encoding="utf-8") + return stub_drift(stub_text=stub_text, registered=registered) diff --git a/scripts/parse_guard/__init__.py b/scripts/parse_guard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/_helpers/__init__.py b/scripts/parse_guard/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/_helpers/scan.py b/scripts/parse_guard/_helpers/scan.py new file mode 100644 index 0000000000..4693d761cf --- /dev/null +++ b/scripts/parse_guard/_helpers/scan.py @@ -0,0 +1,85 @@ +"""Find Polyglot and SQLQuack shared or Snowflake parses that use a library default depth.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from scripts.parse_guard.constants import ( + CRATES_DIRECTORY, + DEFAULT_GUARD_PATTERNS, + GUARD_FIELD, + GUARDED_OPTION_LITERALS, + SKIPPED_DIRECTORIES, + SKIPPED_FILES, + SOURCE_SUFFIX, + TYPE_POSITION_SUFFIXES, +) + +_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(pattern, re.DOTALL) for pattern in DEFAULT_GUARD_PATTERNS +) + + +def crate_sources(*, root: Path) -> tuple[Path, ...]: + """Every non-test Rust source under the workspace crates, except the shared guard.""" + sources: list[Path] = [] + path: Path + for path in (root / CRATES_DIRECTORY).rglob(f"*{SOURCE_SUFFIX}"): + skipped: bool = bool(SKIPPED_DIRECTORIES.intersection(path.relative_to(root).parts)) + if path.name not in SKIPPED_FILES and not skipped: + sources.append(path) + return tuple(sorted(sources)) + + +def unguarded_sites(*, label: str, text: str) -> tuple[str, ...]: + """Each place in `text` a parse uses a default guard, as `label:line pattern`.""" + sites: list[str] = [] + pattern: re.Pattern[str] + for pattern in _PATTERNS: + sites.extend( + f"{label}:{_line(text=text, offset=found.start())} {pattern.pattern}" + for found in pattern.finditer(text) + ) + literal: str + for literal in GUARDED_OPTION_LITERALS: + sites.extend( + f"{label}:{_line(text=text, offset=found.start())} {literal}" + for found in re.finditer(re.escape(literal), text) + if _unguarded_literal(text=text, start=found.start(), literal=literal) + ) + return tuple(sites) + + +def get_parse_guard_errors(root: Path) -> tuple[str, ...]: + """Every unguarded parse site of the workspace crates.""" + errors: list[str] = [] + path: Path + for path in crate_sources(root=root): + errors.extend( + unguarded_sites( + label=path.relative_to(root).as_posix(), text=path.read_text(encoding="utf-8") + ) + ) + return tuple(errors) + + +def _line(*, text: str, offset: int) -> int: + return text.count("\n", 0, offset) + 1 + + +def _unguarded_literal(*, text: str, start: int, literal: str) -> bool: + if text[:start].rstrip().endswith(TYPE_POSITION_SUFFIXES): + return False + return GUARD_FIELD not in _block(text=text, open_at=start + len(literal) - 1) + + +def _block(*, text: str, open_at: int) -> str: + depth: int = 0 + offset: int + character: str + for offset, character in enumerate(text[open_at:]): + depth += {"{": 1, "}": -1}.get(character, 0) + if depth == 0: + return text[open_at : open_at + offset + 1] + return text[open_at:] diff --git a/scripts/parse_guard/constants.py b/scripts/parse_guard/constants.py new file mode 100644 index 0000000000..d2408518d6 --- /dev/null +++ b/scripts/parse_guard/constants.py @@ -0,0 +1,24 @@ +"""Text that marks a guarded parse using a library's default depth.""" + +CRATES_DIRECTORY: str = "crates" +SOURCE_SUFFIX: str = ".rs" +SKIPPED_DIRECTORIES: frozenset[str] = frozenset({"tests", "target", "parse_guard"}) +SKIPPED_FILES: frozenset[str] = frozenset({"tests.rs"}) +DEFAULT_GUARD_PATTERNS: tuple[str, ...] = ( + r"ParseOptions::default\(\)", + r"complexity_guard: None", + r"maxFunctionCallDepth", + r"max_function_call_depth: Some\((?!MAX_FUNCTION_CALL_DEPTH\))", + r"const MAX_FUNCTION_CALL_DEPTH", + r"polyglot_sql::(?:parse|transpile|validate)\(", + r"\bparse_data_type\(", + r"Dialect::get(?:_by_name)?\([^;]*?\)\s*(?:\.[a-z_]+\([^;]*?\)\??\s*)*?\.parse\(", +) +GUARDED_OPTION_LITERALS: tuple[str, ...] = ( + "SchemaValidationOptions {", + "AnalyzeQueryOptions {", + "ValidationOptions {", + "ParseOptions {", +) +GUARD_FIELD: str = "complexity_guard" +TYPE_POSITION_SUFFIXES: tuple[str, ...] = ("->", "struct") diff --git a/scripts/parse_guard/main/__init__.py b/scripts/parse_guard/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/main/check.py b/scripts/parse_guard/main/check.py new file mode 100644 index 0000000000..b66ae63f97 --- /dev/null +++ b/scripts/parse_guard/main/check.py @@ -0,0 +1,19 @@ +"""Parse guard command entrypoint.""" + +from __future__ import annotations + +from scripts.parse_guard._helpers.scan import get_parse_guard_errors +from scripts.root_check.main.run_root_check import run_root_check + + +def run_parse_guard_check(argv: list[str] | None = None) -> int: + """Fail when a Polyglot or SQLQuack shared or Snowflake parse uses a library default depth.""" + return run_root_check( + argv=argv, + description=__doc__, + errors_under=get_parse_guard_errors, + messages=( + "Parse with sqlbuild_analysis::parse_guard's options, not a library default:", + "Polyglot and SQLQuack's shared and Snowflake parsers all use the shared guard.", + ), + ) diff --git a/scripts/release_gate/main/check.py b/scripts/release_gate/main/check.py index 11194a2fb7..6d7c916a82 100644 --- a/scripts/release_gate/main/check.py +++ b/scripts/release_gate/main/check.py @@ -2,27 +2,28 @@ from __future__ import annotations -import argparse -import sys from pathlib import Path from scripts.release_gate._helpers.gate import get_release_gate_errors from scripts.release_gate.exceptions import ReleaseGateError +from scripts.root_check.main.run_root_check import run_root_check def run_release_gate_check(argv: list[str] | None = None) -> int: """Fail when anything but the trusted release checks can pass a release pull request.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_gate_errors, + messages=( + "Release pull request Verify gate is invalid:", + "Release pull request Verify gate is valid.", + ), + ) + + +def _gate_errors(root: Path) -> tuple[str, ...]: try: - errors: tuple[str, ...] = get_release_gate_errors(arguments.root) + return get_release_gate_errors(root) except ReleaseGateError as error: - errors = (str(error),) - if errors: - print("Release pull request Verify gate is invalid:", file=sys.stderr) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Release pull request Verify gate is valid.") - return 0 + return (str(error),) diff --git a/scripts/root_check/__init__.py b/scripts/root_check/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/root_check/main/__init__.py b/scripts/root_check/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/root_check/main/run_root_check.py b/scripts/root_check/main/run_root_check.py new file mode 100644 index 0000000000..203b68c17a --- /dev/null +++ b/scripts/root_check/main/run_root_check.py @@ -0,0 +1,30 @@ +"""Run one repository check over `--root` and report its errors.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Callable +from pathlib import Path + + +def run_root_check( + *, + argv: list[str] | None, + description: str | None, + errors_under: Callable[[Path], tuple[str, ...]], + messages: tuple[str, str], +) -> int: + """Print the check's errors under the failure heading and fail, or print the success line.""" + failure_heading, success_line = messages + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=description) + parser.add_argument("--root", type=Path, default=Path.cwd()) + arguments: argparse.Namespace = parser.parse_args(argv) + errors: tuple[str, ...] = errors_under(arguments.root) + if errors: + print(failure_heading, file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(success_line) + return 0 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 47b8971951..2626a77c8d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -227,6 +227,7 @@ CURRENT_EVENT_LOG_SCHEMA_VERSION: int CURRENT_RUN_STORAGE_SCHEMA_VERSION: int COMPUTE_LOG_FORMAT_VERSION: int DEFAULT_RETENTION_COUNT: int +MAX_FUNCTION_CALL_DEPTH: int MAX_READ_BYTES: int CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int MIGRATION_WRITE_ATTEMPTS: int diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index d68a9dc000..6f78551182 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -2,10 +2,12 @@ from __future__ import annotations +from sqlbuild._native import MAX_FUNCTION_CALL_DEPTH + POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 POLYGLOT_COMPLEXITY_GUARD_PARAMETER: str = "complexity_guard" POLYGLOT_COMPLEXITY_GUARD_OPTION: str = "complexityGuard" POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION: str = "maxFunctionCallDepth" -POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 +POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = MAX_FUNCTION_CALL_DEPTH diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 87177244e5..f230c7979c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -47,6 +47,15 @@ class SnowflakeBodyShapeCase: expected_format_exit: int = 0 +@dataclass(frozen=True) +class SharedDepthModelRuleCase: + description: str + template: str + rule: str + expected_evaluated_models: int + expected_unevaluated_resources: int + + @dataclass(frozen=True) class UnevaluatedResourceCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index 4939d1b9fc..0ac0eb5ef6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -11,6 +11,7 @@ EvaluatedModelRulesCase, EvaluatedPredicateCase, GroupedRulesCase, + SharedDepthModelRuleCase, SnowflakeBodyShapeCase, UnevaluatedResourceCase, UnevaluatedRuleCase, @@ -48,14 +49,6 @@ ), ), ), - UnevaluatedResourceCase( - "built-in model rule beyond the parser budget", - "models/staging/orders.sql", - "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", - expected_evaluated_models=0, - expected_reason="could not parse models/staging/orders.sql for rules", - rule="SQBRMODEL101", - ), UnevaluatedResourceCase( "SQL-test guard", "tests/unit/test_orders.sql", @@ -109,6 +102,40 @@ def test_given_guarded_resource_when_running_rules_then_reports_failure_and_cove ) +@pytest.mark.parametrize( + "test_case", + [ + SharedDepthModelRuleCase( + "a built-in model rule parses calls nested as deep as the compiler accepts", + "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", + "SQBRMODEL101", + expected_evaluated_models=1, + expected_unevaluated_resources=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_calls_nested_within_the_shared_depth_when_running_model_rules_then_evaluates( + test_case: SharedDepthModelRuleCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + resource_path="models/staging/orders.sql", + resource_template=test_case.template, + selected_rules=(test_case.rule,), + ) + + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", test_case.rule + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["unevaluated_resources"] == test_case.expected_unevaluated_resources + assert payload["evaluated_models"] == test_case.expected_evaluated_models + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/scripts/parse_guard/__init__.py b/tests/unit/scripts/parse_guard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/parse_guard/_helpers/__init__.py b/tests/unit/scripts/parse_guard/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/parse_guard/_helpers/_test_types.py b/tests/unit/scripts/parse_guard/_helpers/_test_types.py new file mode 100644 index 0000000000..14a0aafde2 --- /dev/null +++ b/tests/unit/scripts/parse_guard/_helpers/_test_types.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UnguardedSiteTestCase: + description: str + text: str + expected_sites: tuple[str, ...] + + +@dataclass(frozen=True) +class RepositoryParseGuardTestCase: + description: str + expected_errors: tuple[str, ...] diff --git a/tests/unit/scripts/parse_guard/_helpers/test_scan.py b/tests/unit/scripts/parse_guard/_helpers/test_scan.py new file mode 100644 index 0000000000..82952a40f4 --- /dev/null +++ b/tests/unit/scripts/parse_guard/_helpers/test_scan.py @@ -0,0 +1,96 @@ +from pathlib import Path + +import pytest + +from scripts.parse_guard._helpers.scan import get_parse_guard_errors, unguarded_sites +from tests.unit.scripts.parse_guard._helpers._test_types import ( + RepositoryParseGuardTestCase, + UnguardedSiteTestCase, +) + +_REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[5] + + +@pytest.mark.parametrize( + "test_case", + [ + UnguardedSiteTestCase( + description="a bare Polyglot parse across lines", + text="fn a() {\n Dialect::get(d)\n .parse(sql)\n}\n", + expected_sites=( + r"x.rs:2 Dialect::get(?:_by_name)?\([^;]*?\)\s*(?:\.[a-z_]+\([^;]*?\)\??\s*)*?" + r"\.parse\(", + ), + ), + UnguardedSiteTestCase( + description="SQLQuack's default options", + text="fn b() { parse_strict(sql, d, &ParseOptions::default()); }", + expected_sites=(r"x.rs:1 ParseOptions::default\(\)",), + ), + UnguardedSiteTestCase( + description="a local depth instead of the shared constant", + text="fn c() { Guard { max_function_call_depth: Some(512) } }", + expected_sites=( + r"x.rs:1 max_function_call_depth: Some\((?!MAX_FUNCTION_CALL_DEPTH\))", + ), + ), + UnguardedSiteTestCase( + description="an analysis options literal without a guard", + text="fn d() { let o = AnalyzeQueryOptions { dialect, schema: Some(S { t }) }; }", + expected_sites=("x.rs:1 AnalyzeQueryOptions {",), + ), + UnguardedSiteTestCase( + description="Polyglot parse options that fill the rest from the default", + text="fn h() { let o = ParseOptions { ..Default::default() }; }", + expected_sites=("x.rs:1 ParseOptions {",), + ), + UnguardedSiteTestCase( + description="a data type parse without options", + text="fn i() { Dialect::get(d).parse_data_type(spelling) }", + expected_sites=(r"x.rs:1 \bparse_data_type\(",), + ), + UnguardedSiteTestCase( + description=( + "SQLQuack's per-dialect parsers are excluded: they take no options and keep their " + "own depth limits until SQLQuack accepts options for them" + ), + text=( + "fn k() { sqlquack_syntax_postgres::parse_strict(sql) }\n" + "fn l() { sqlquack_syntax_bigquery::parse_type(spelling) }\n" + "fn m(parser: &dyn NativeDialect) { parser.parse(sql) }\n" + ), + expected_sites=(), + ), + UnguardedSiteTestCase( + description="the shared options and constant", + text=( + "fn e() -> AnalyzeQueryOptions { AnalyzeQueryOptions { complexity_guard: g } }\n" + "fn f() { Dialect::get(d).parse_with_options(sql, &polyglot_parse_options()); }\n" + "fn g() { R { max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH) } }\n" + "fn j() { d.parse_data_type_with_options(t, &ParseOptions { complexity_guard: g }) }\n" + ), + expected_sites=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_parse_text_when_scanning_then_reports_each_default_guarded_site( + test_case: UnguardedSiteTestCase, +) -> None: + assert unguarded_sites(label="x.rs", text=test_case.text) == test_case.expected_sites + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryParseGuardTestCase( + description="every workspace parse uses the shared guard", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_when_checking_parse_guard_then_returns_expected_errors( + test_case: RepositoryParseGuardTestCase, +) -> None: + assert get_parse_guard_errors(_REPOSITORY_ROOT) == test_case.expected_errors