From 83d6a9cd0b721ab76f6cf844f9328bc39f8873aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:48:47 +0000 Subject: [PATCH 1/5] fix(analysis): allow function calls nested 256 deep on every SQL parse A Snowflake SQL macro whose body nests 65 REPLACE calls failed to compile with P015 "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value 65 exceeds configured limit 64". The native Snowflake module, which parses macro bodies and models since v0.208.0, passed SQLQuack's default options, so the library's 64-call guard applied. Other parse paths set their own limit (128 in query facts and Polyglot query analysis, 512 elsewhere) or none, so the same SQL passed one command and failed another. Every Polyglot and SQLQuack parse now takes its options from sqlbuild_analysis::parse_guard, whose one MAX_FUNCTION_CALL_DEPTH is 256, as SQLQuack's own guarded parse uses. Python's Polyglot proxy reads the same constant from the native module. Both parsers grow their stack on demand and keep their parser-depth (1,024) and AST-depth (512) limits; tests parse 256 nested calls with each guarded parser on a 2 MiB thread and check one more names the limit. A macro nested past it still fails with P015 naming the model, both lines and limit 256. `make check-ci` gains check-parse-guard, which fails when a crate source parses with a library default: ParseOptions::default(), an option literal without complexity_guard, a bare Polyglot parse, or a local depth. SQLClean rules parse inside sqlquack-lint with its own limit of 128, which SQLBuild cannot set. --- Makefile | 8 +- .../_helpers/dialects/snowflake_dialect.rs | 5 +- .../analysis_session/_helpers/dict_walk.rs | 19 ++--- .../_helpers/dynamic_pivot.rs | 5 +- .../analysis_session/_helpers/enrichment.rs | 7 +- .../assembly/analysis_session/constants.rs | 2 - .../src/assembly/project/_helpers/syntax.rs | 13 +-- .../src/assembly/project/constants.rs | 1 - .../column_references/_helpers/references.rs | 5 +- .../_helpers/polyglot/parse_options.rs | 12 +-- .../src/contracts/_helpers/enum_literals.rs | 5 +- .../src/diff_selection/constants.rs | 1 - .../diff_selection/main/raw_query_shape.rs | 14 +-- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/lineage/_helpers/parsed_lineage.rs | 12 +-- .../src/lineage/_helpers/rich_lineage.rs | 8 +- .../src/lineage/constants.rs | 2 - .../src/parse_guard/constants.rs | 4 + .../src/parse_guard/main/mod.rs | 3 + .../main/polyglot_complexity_guard.rs | 14 +++ .../main/polyglot_parse_options.rs | 13 +++ .../main/sqlquack_parse_options.rs | 16 ++++ .../sqlbuild-analysis/src/parse_guard/mod.rs | 7 ++ .../src/parse_guard/tests.rs | 6 ++ .../src/parse_guard/tests/helpers.rs | 70 +++++++++++++++ .../src/parse_guard/tests/test_call_depth.rs | 55 ++++++++++++ .../src/parse_guard/tests/test_types.rs | 6 ++ .../src/planner_fixtures/main/star_keys.rs | 3 +- .../_helpers/proofs/parse.rs | 15 ++-- .../src/query_analysis/_helpers/engine.rs | 20 ++--- .../src/relation_names/_helpers/walk.rs | 17 ++-- .../src/relation_names/constants.rs | 2 - .../main/resolve_required_columns.rs | 5 +- .../_helpers/sql_text/parsed_sql.rs | 17 +--- .../src/semantic_checks/constants.rs | 1 - .../src/semantic_usage/main.rs | 5 +- .../semantic_validation/_helpers/catalog.rs | 9 +- .../_helpers/diagnostics.rs | 6 +- .../src/semantic_validation/main.rs | 14 ++- .../src/snowflake_cost/_helpers/statements.rs | 10 +-- .../sqlbuild-analysis/src/sql_macros/tests.rs | 2 + .../src/sql_macros/tests/helpers.rs | 15 ++++ .../src/sql_macros/tests/test_call_depth.rs | 45 ++++++++++ .../src/sql_macros/tests/test_types.rs | 7 ++ .../src/sql_tree/_helpers/parse.rs | 5 +- .../warehouse_types/_helpers/type_trees.rs | 5 +- .../_helpers/exported_constants/constants.rs | 2 + .../src/lint_text/_helpers/format_bodies.rs | 5 +- .../sqlbuild-rules/src/lint_text/constants.rs | 1 - .../src/query_facts/_helpers/facts.rs | 15 +--- .../src/query_facts/constants.rs | 1 - scripts/check_parse_guard.py | 14 +++ scripts/parse_guard/__init__.py | 0 scripts/parse_guard/_helpers/__init__.py | 0 scripts/parse_guard/_helpers/scan.py | 85 +++++++++++++++++++ scripts/parse_guard/constants.py | 22 +++++ scripts/parse_guard/main/__init__.py | 0 scripts/parse_guard/main/check.py | 27 ++++++ src/sqlbuild/_native.pyi | 1 + .../compiler/sql_analysis/constants.py | 4 +- tests/unit/scripts/parse_guard/__init__.py | 0 .../scripts/parse_guard/_helpers/__init__.py | 0 .../parse_guard/_helpers/_test_types.py | 14 +++ .../scripts/parse_guard/_helpers/test_scan.py | 73 ++++++++++++++++ 64 files changed, 624 insertions(+), 157 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/parse_guard/constants.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/mod.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/tests.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs create mode 100644 crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs create mode 100644 crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs create mode 100644 scripts/check_parse_guard.py create mode 100644 scripts/parse_guard/__init__.py create mode 100644 scripts/parse_guard/_helpers/__init__.py create mode 100644 scripts/parse_guard/_helpers/scan.py create mode 100644 scripts/parse_guard/constants.py create mode 100644 scripts/parse_guard/main/__init__.py create mode 100644 scripts/parse_guard/main/check.py create mode 100644 tests/unit/scripts/parse_guard/__init__.py create mode 100644 tests/unit/scripts/parse_guard/_helpers/__init__.py create mode 100644 tests/unit/scripts/parse_guard/_helpers/_test_types.py create mode 100644 tests/unit/scripts/parse_guard/_helpers/test_scan.py diff --git a/Makefile b/Makefile index d45aa2ba11..1b5cb187ea 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash -.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ +.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-parse-guard check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \ check-e2e-shards check-test-shards test-shard-core test-shard-publication test-shard-history \ test-e2e-duckdb test-e2e-duckdb-build-core \ test-e2e-duckdb-build-incremental \ @@ -747,7 +747,7 @@ verify-quick: exit $$status -check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint +check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-parse-guard check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint uv run ruff format --check . uv run ruff check . uv run ty check src tests @@ -770,6 +770,10 @@ check-native-stub: uv run python -m scripts.check_native_stub +check-parse-guard: + uv run python -m scripts.check_parse_guard + + check-build-features: uv run python -m scripts.check_build_features diff --git a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs index 514da50e62..adc88494de 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/_helpers/dialects/snowflake_dialect.rs @@ -5,7 +5,7 @@ use std::num::IntErrorKind; use sqlquack_parse::{StrictFailure, parse_strict}; use sqlquack_semantics::dialects::constants::{SNOWFLAKE, SNOWFLAKE_QUOTED_IGNORING_CASE}; use sqlquack_semantics::dialects::types::DialectProfile; -use sqlquack_syntax::{Dialect, Error, Expression, ParseOptions, Parsed, TokenizeFailure}; +use sqlquack_syntax::{Dialect, Error, Expression, Parsed, TokenizeFailure}; use crate::analysis_route::_helpers::dialects::engine_types::snowflake_engine_type; use crate::analysis_route::_helpers::dialects::type_spelling::respelt; @@ -14,6 +14,7 @@ use crate::analysis_route::constants::{ }; use crate::analysis_route::models::{AnalysisInput, ParserGap, SnowflakeDialect}; use crate::analysis_route::types::NativeDialect; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::type_system::main::normalize_text_type::normalize_text_type; use crate::type_system::models::{NormalizedType, TypeDialect, TypeFamily}; @@ -67,7 +68,7 @@ impl NativeDialect for SnowflakeDialect { /// Every statement the shared parser reads from `sql`, or its first refusal, never marked syntax. pub(crate) fn shared_parsed(sql: &str, dialect: Dialect) -> Result { - let statements: Vec = parse_strict(sql, dialect, &ParseOptions::default()) + let statements: Vec = parse_strict(sql, dialect, &sqlquack_parse_options()) .map_err(|failure| match failure { StrictFailure::Tokens( TokenizeFailure::Uncovered { start, .. } | TokenizeFailure::Overlap { start, .. }, diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs index 9602a08aab..155cb4940b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs @@ -2,20 +2,20 @@ use polyglot_sql::expressions::Cte; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions, ast_json}; -use serde_json::{Map, Value, json}; +use polyglot_sql::{DataType, Dialect, Expression, ParseOptions, ast_json}; +use serde_json::{Map, Value}; use crate::assembly::analysis_session::models::ProjectionFacts; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::relation_markers::constants::MARKER_NAME_PREFIX; use crate::assembly::analysis_session::constants::{ ALIAS_AST_KIND, ANNOTATED_AST_KIND, ARRAY_FIXTURE_TYPE, ARRAY_PAYLOAD_KIND, CATALOG_TABLE_ARG, COLUMN_AST_KIND, CTE_ALIAS_PAYLOAD_KEY, CTE_LIST_PAYLOAD_KEY, DEPENDENCY_FUNCTION_NAMES, FUNCTION_AST_KIND, FUNCTION_FIXTURE_TYPES, FUNCTION_PAYLOAD_KIND, HELPER_READS_DIALECT, - LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, MAX_FUNCTION_CALL_DEPTH, NAME_PAYLOAD_KEY, - NULL_PAYLOAD_KIND, RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY, - SET_OPERATION_PAYLOAD_KEYS, STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND, - VARIANT_FIXTURE_TYPE, WILDCARD, + LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, NAME_PAYLOAD_KEY, NULL_PAYLOAD_KIND, + RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY, SET_OPERATION_PAYLOAD_KEYS, + STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND, VARIANT_FIXTURE_TYPE, WILDCARD, }; /// Python's `bool(value)`. @@ -193,12 +193,7 @@ pub(crate) fn parse_one_or_error( sql: &str, dialect: &str, ) -> Result, String> { - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| error.to_string())?; - let options: ParseOptions = ParseOptions { - complexity_guard: Some(guard), - }; + let options: ParseOptions = polyglot_parse_options(); let parsed: Result, _> = Dialect::get_by_name(dialect) .ok_or_else(|| format!("unknown dialect {dialect}"))? .parse_with_options(sql, &options); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index 7323c2820a..c6ac5aaec3 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; -use sqlquack_syntax::{CallMode, Dialect, Expression, ParseOptions}; +use sqlquack_syntax::{CallMode, Dialect, Expression}; use crate::assembly::analysis_session::_helpers::dict_walk::{ casefold, column_name, dict_list, identifier_name, is_single_wildcard, nested, node_key, @@ -25,6 +25,7 @@ use crate::assembly::analysis_session::models::{ ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables, }; use crate::assembly::analysis_session::types::Shapes; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::relation_markers::constants::MARKER_NAME_PREFIX; use crate::relation_markers::main::markers_as_names::markers_as_names; use crate::relation_markers::types::MarkerCall; @@ -134,7 +135,7 @@ fn parsed(sql: &str, dialect: &str) -> Result, String> { let parsed = sqlquack_parse::parse_statements_with( &named, dialect, - &ParseOptions::default(), + &sqlquack_parse_options(), CallMode::Faithful, ); let mut statements: Vec = match parsed { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index 12bab7d107..e29815c6ff 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -14,14 +14,15 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{truthy, upper}; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::constants::{ CAST_TRANSFORM, CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, FILTER_CONTEXT, - MAX_FUNCTION_CALL_DEPTH, NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE, - SELECT_SHAPE, SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, + NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE, SELECT_SHAPE, + SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT, TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, TRANSFORM_STAR, UNKNOWN_NULLABILITY, UNKNOWN_TYPE, WILDCARD, }; use crate::assembly::analysis_session::models::NullabilityCallback; use crate::assembly::analysis_session::models::{ColumnFact, LineageRow, ModelRequest}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::NormalizationInput; @@ -77,7 +78,7 @@ fn analysed(input: &EnrichmentInput<'_>) -> Result { } options.insert( "complexityGuard".to_owned(), - json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH}), + serde_json::to_value(polyglot_complexity_guard()).map_err(|error| error.to_string())?, ); let options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options)) .map_err(|error| format!("query analysis options: {error}"))?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 968ee107ea..70ca4a1333 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -56,8 +56,6 @@ pub(crate) const FILTER_CONTEXT: &str = "filter"; pub(crate) const NULL_KEYWORD: &str = "NULL"; pub(crate) const RESOLVED_SOURCE_CONFIDENCE: &str = "resolved"; pub(crate) const WILDCARD: &str = "*"; -/// SQLBuild's `analyze_query` complexity guard. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; /// The dialects whose dynamic pivots the compiler proves, and those with DuckDB's PIVOT form. pub(crate) const SUPPORTED_PIVOT_DIALECTS: [&str; 3] = ["duckdb", "motherduck", "snowflake"]; pub(crate) const SIMPLIFIED_PIVOT_DIALECTS: [&str; 2] = ["duckdb", "motherduck"]; diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs index b9706b03d1..4e19530a98 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs @@ -2,12 +2,10 @@ use std::collections::HashMap; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, ParseOptions, ValidationOptions, - validate_with_dialect, -}; +use polyglot_sql::{Dialect, DialectType, ParseOptions, ValidationOptions, validate_with_dialect}; -use crate::assembly::project::constants::{INVALID_SQL, MAX_FUNCTION_CALL_DEPTH}; +use crate::assembly::project::constants::INVALID_SQL; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode}; @@ -40,10 +38,7 @@ pub(crate) fn syntax_error( }; let dialect_type: DialectType = parser_dialect(Some(dialect)) .map_err(|unknown| SyntaxFailure::UnknownDialect(unknown.to_owned()))?; - let complexity_guard = ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..Default::default() - }; + let complexity_guard = polyglot_complexity_guard(); let polyglot: Dialect = Dialect::get(dialect_type); Ok(match mode { SyntaxMode::Validate => { diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index 91b7a61a5e..6d63fc9a7c 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -16,7 +16,6 @@ pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; pub(crate) const PYTHON_NONE: &str = "None"; pub(crate) const PYTHON_TRUE: &str = "True"; pub(crate) const PYTHON_FALSE: &str = "False"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name"; pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database"; pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema"; diff --git a/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs b/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs index 2282e89ede..1d001868bb 100644 --- a/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs +++ b/crates/sqlbuild-analysis/src/column_references/_helpers/references.rs @@ -1,5 +1,6 @@ //! Resolve every reference to one relation column, with the spans a rename rewrites. +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use polyglot_sql::{ Dialect, DialectType, Expression, MappingSchema, Resolver, Scope, ValidationSchema, build_scope, mapping_schema_from_validation_schema_with_dialect, walk_in_scope, @@ -109,7 +110,9 @@ pub(crate) fn analyze_json(request_json: &str) -> Result { request.dialect ) })?; - let Ok(statements) = Dialect::get(dialect).parse(&request.sql) else { + let Ok(statements) = + Dialect::get(dialect).parse_with_options(&request.sql, &polyglot_parse_options()) + else { return serde_json::to_string(&ReferenceResponse::default()) .map_err(|error| error.to_string()); }; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs index 773f7acad5..3a1b66f829 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs @@ -1,15 +1,9 @@ //! The Polyglot parse options SQLBuild's wheel proxy applies to every parse. -use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; - -use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; +use polyglot_sql::ParseOptions; /// The proxy's complexity guard, decoded as the wheel decodes it. pub(crate) fn guarded_parse_options() -> Result { - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - }))?; - Ok(ParseOptions { - complexity_guard: Some(guard), - }) + Ok(polyglot_parse_options()) } diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs index adaf9e1f93..5a021d54c4 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/enum_literals.rs @@ -5,6 +5,7 @@ use polyglot_sql::tokens::{Token, TokenType}; use polyglot_sql::{Dialect, DialectType, Expression}; use crate::contracts::models::LiteralLocation; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// How deep the walk follows set operations and passthrough columns before it stops. const MAX_DEPTH: usize = 64; @@ -69,7 +70,9 @@ pub(crate) fn parsed_query(sql: &str, dialect: &str) -> Option { let Ok(dialect) = dialect.parse::() else { return None; }; - let Ok(mut statements) = polyglot_sql::parse(sql, dialect) else { + let Ok(mut statements) = + polyglot_sql::parse_with_options(sql, dialect, &polyglot_parse_options()) + else { return None; }; (statements.len() == 1).then(|| statements.pop()).flatten() diff --git a/crates/sqlbuild-analysis/src/diff_selection/constants.rs b/crates/sqlbuild-analysis/src/diff_selection/constants.rs index 13daec5375..ad63030eee 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/constants.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/constants.rs @@ -13,4 +13,3 @@ pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; pub(crate) const RAW_QUERY_READ_ONLY_ROOTS: [&str; 5] = ["except", "intersect", "select", "union", "values"]; -pub(crate) const RAW_QUERY_GUARD: &str = r#"{"maxFunctionCallDepth": 512}"#; diff --git a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs index c553e213e1..6e879e1e36 100644 --- a/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs +++ b/crates/sqlbuild-analysis/src/diff_selection/main/raw_query_shape.rs @@ -1,25 +1,19 @@ //! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape. +use polyglot_sql::Expression; use polyglot_sql::dialects::Dialect; use polyglot_sql::traversal::ExpressionWalk; -use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions}; -use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS}; +use crate::diff_selection::constants::RAW_QUERY_READ_ONLY_ROOTS; use crate::diff_selection::models::RawQueryShape; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// The query's shape, or the text of the exception the wheel raises for it. pub fn raw_query_shape(sql: &str, dialect: &str) -> Result { let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?; - let guard: ComplexityGuardOptions = - serde_json::from_str(RAW_QUERY_GUARD).map_err(|error| error.to_string())?; let mut statements: Vec = parser - .parse_with_options( - sql, - &ParseOptions { - complexity_guard: Some(guard), - }, - ) + .parse_with_options(sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; if statements.len() != 1 { return Err(format!("Expected 1 statement, found {}", statements.len())); diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 0f0ca871f0..14f19e6e7b 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -26,6 +26,7 @@ pub mod lineage_graph; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; +pub mod parse_guard; pub mod planner_changes; pub mod planner_fixtures; pub mod planner_identity; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs index 93f982d537..bce222927c 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet}; -use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, Expression, ParseOptions}; +use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions}; use crate::lineage::_helpers::projections::{ UnreadableExpression, classify_transform, is_star, projection_output_name, @@ -10,20 +10,16 @@ use crate::lineage::_helpers::projections::{ }; use crate::lineage::_helpers::references::{PhysicalResource, normalized_sql, physical_resources}; use crate::lineage::_helpers::stars::star_lineage; -use crate::lineage::constants::{KIND_SELECT, KIND_UNION, MAX_FUNCTION_CALL_DEPTH}; +use crate::lineage::constants::{KIND_SELECT, KIND_UNION}; use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSource, LineageTransformKind, }; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// The parse options SQLBuild's Polyglot proxy gives `parse_one`, decoded as the wheel does. pub(crate) fn proxy_parse_options() -> Result { - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - }))?; - Ok(ParseOptions { - complexity_guard: Some(guard), - }) + Ok(polyglot_parse_options()) } pub(crate) struct ParsedModel<'a> { diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs index 302db0fdb6..8ec0387fe7 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs @@ -15,12 +15,13 @@ use crate::lineage::_helpers::references::{ PhysicalResource, normalized_sql, physical_resource_name, physical_resources, }; use crate::lineage::_helpers::stars::star_lineage; -use crate::lineage::constants::{MAX_FUNCTION_CALL_DEPTH, STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE}; +use crate::lineage::constants::{STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE}; use crate::lineage::models::{ LineageColumn, LineageConfidence, LineageNullability, LineageResourceType, LineageSchemaResource, LineageSource, LineageTransformKind, RichLineageColumn, RichLineageOutcome, RichSchemaResource, }; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; /// What every model of one request shares: the dialect, the guard and the known tables. pub(crate) struct RichContext { @@ -42,10 +43,7 @@ impl RichContext { ) -> Self { Self { dialect, - guard: ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..ComplexityGuardOptions::default() - }, + guard: polyglot_complexity_guard(), tables: schema_tables(schema, referenced), names, } diff --git a/crates/sqlbuild-analysis/src/lineage/constants.rs b/crates/sqlbuild-analysis/src/lineage/constants.rs index cf37260374..af4ceac67e 100644 --- a/crates/sqlbuild-analysis/src/lineage/constants.rs +++ b/crates/sqlbuild-analysis/src/lineage/constants.rs @@ -21,8 +21,6 @@ pub(crate) const PAYLOAD_COLUMN: &str = "column"; pub(crate) const PAYLOAD_NAME: &str = "name"; pub(crate) const PAYLOAD_TABLE: &str = "table"; pub(crate) const SET_OPERATION_SIDES: [&str; 2] = ["left", "right"]; -/// The function-call depth SQLBuild's Polyglot proxy passes on every parse. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const REFERENCE_PREFIX: &str = "__"; pub(crate) const UDF_CALL: &str = "__udf("; pub(crate) const PHYSICAL_RESOURCE_PREFIX: &str = "__sqlbuild_"; diff --git a/crates/sqlbuild-analysis/src/parse_guard/constants.rs b/crates/sqlbuild-analysis/src/parse_guard/constants.rs new file mode 100644 index 0000000000..db45e90082 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/constants.rs @@ -0,0 +1,4 @@ +//! Constants of parse_guard. + +/// The deepest function-call nesting any SQLBuild parse accepts, as SQLQuack's guarded parse. +pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs new file mode 100644 index 0000000000..793cfb6cd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/mod.rs @@ -0,0 +1,3 @@ +pub mod polyglot_complexity_guard; +pub mod polyglot_parse_options; +pub mod sqlquack_parse_options; diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs new file mode 100644 index 0000000000..1114b6627d --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_complexity_guard.rs @@ -0,0 +1,14 @@ +//! `polyglot_complexity_guard`: Polyglot's default limits with SQLBuild's function-call depth. + +use polyglot_sql::ComplexityGuardOptions; + +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; + +/// Polyglot's default complexity limits, allowing SQLBuild's function-call depth. +#[must_use] +pub fn polyglot_complexity_guard() -> ComplexityGuardOptions { + ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs new file mode 100644 index 0000000000..05e0e6760d --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/polyglot_parse_options.rs @@ -0,0 +1,13 @@ +//! `polyglot_parse_options`: the options every SQLBuild Polyglot parse passes. + +use polyglot_sql::ParseOptions; + +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; + +/// Polyglot parse options under SQLBuild's complexity guard. +#[must_use] +pub fn polyglot_parse_options() -> ParseOptions { + ParseOptions { + complexity_guard: Some(polyglot_complexity_guard()), + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs b/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs new file mode 100644 index 0000000000..7feafefee6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/main/sqlquack_parse_options.rs @@ -0,0 +1,16 @@ +//! `sqlquack_parse_options`: the options every SQLBuild SQLQuack parse passes. + +use sqlquack_syntax::{ComplexityGuardOptions, ParseOptions}; + +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; + +/// SQLQuack's default complexity limits, allowing SQLBuild's function-call depth. +#[must_use] +pub fn sqlquack_parse_options() -> ParseOptions { + ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + }), + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/mod.rs new file mode 100644 index 0000000000..add9304504 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/mod.rs @@ -0,0 +1,7 @@ +//! The one function-call depth every SQL parse in SQLBuild allows, for Polyglot and SQLQuack alike. + +pub mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests.rs b/crates/sqlbuild-analysis/src/parse_guard/tests.rs new file mode 100644 index 0000000000..6d2a051968 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_call_depth.rs"] +mod test_call_depth; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs new file mode 100644 index 0000000000..13be5c2772 --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/helpers.rs @@ -0,0 +1,70 @@ +use polyglot_sql::DialectType; +use sqlquack_syntax::{CallMode, Dialect}; + +use crate::analysis_route::models::SnowflakeDialect; +use crate::analysis_route::types::NativeDialect; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; + +/// A query selecting `depth` nested `REPLACE` calls around one column. +pub(super) fn nested_calls(depth: usize) -> String { + let closings: String = (0..depth) + .map(|level| format!(", 'a{level}', 'b')")) + .collect(); + format!( + "SELECT {}name{closings} AS cleaned FROM races", + "REPLACE(".repeat(depth) + ) +} + +/// `work`'s result on a thread with Rust's default 2 MiB stack, as rayon and `std::thread` use. +pub(super) fn on_default_stack(work: impl FnOnce() -> T + Send) -> T { + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(2 * 1024 * 1024) + .spawn_scoped(scope, work) + .expect("thread starts") + .join() + .expect("parse does not overflow the stack") + }) +} + +/// Each parser SQLBuild guards, with whether it accepts `sql` or the error it raised. +pub(super) fn parses(sql: &str) -> Vec<(&'static str, Result<(), String>)> { + vec![ + ( + "native Snowflake module", + SnowflakeDialect + .parse(sql) + .map(|_| ()) + .map_err(|gap| gap.message), + ), + ( + "SQLQuack Snowflake", + sqlquack_parse::parse_statements_with( + sql, + Dialect::Snowflake, + &sqlquack_parse_options(), + CallMode::Faithful, + ) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ( + "Polyglot Snowflake", + polyglot_sql::parse_with_options( + sql, + DialectType::Snowflake, + &polyglot_parse_options(), + ) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ( + "Polyglot DuckDB", + polyglot_sql::parse_with_options(sql, DialectType::DuckDB, &polyglot_parse_options()) + .map(|_| ()) + .map_err(|error| error.to_string()), + ), + ] +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs new file mode 100644 index 0000000000..f9d812847c --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/test_call_depth.rs @@ -0,0 +1,55 @@ +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::parse_guard::tests::helpers::{nested_calls, on_default_stack, parses}; +use crate::parse_guard::tests::test_types::CallDepthTestCase; + +#[test] +fn given_nested_calls_when_parsing_on_a_default_stack_then_each_parser_applies_the_shared_limit() { + let test_cases = [ + CallDepthTestCase { + description: "one past the libraries' default of 64", + depth: 65, + expected_error: None, + }, + CallDepthTestCase { + description: "a hundred deep", + depth: 100, + expected_error: None, + }, + CallDepthTestCase { + description: "exactly the limit", + depth: MAX_FUNCTION_CALL_DEPTH, + expected_error: None, + }, + CallDepthTestCase { + description: "one past the limit names it", + depth: MAX_FUNCTION_CALL_DEPTH + 1, + expected_error: Some(format!( + "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value {} exceeds configured limit \ + {MAX_FUNCTION_CALL_DEPTH}", + MAX_FUNCTION_CALL_DEPTH + 1 + )), + }, + ]; + + for test_case in test_cases { + let sql: String = nested_calls(test_case.depth); + let expected_length: usize = test_case.expected_error.as_ref().map_or(0, String::len); + + for (parser, outcome) in on_default_stack(|| parses(&sql)) { + let guard_error: Option = outcome.err().map(|error| { + error + .get(error.find("E_GUARD").unwrap_or(0)..) + .unwrap_or_default() + .chars() + .take(expected_length) + .collect() + }); + + assert_eq!( + guard_error, test_case.expected_error, + "{}: {parser}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs b/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs new file mode 100644 index 0000000000..ab236298be --- /dev/null +++ b/crates/sqlbuild-analysis/src/parse_guard/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct CallDepthTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + /// The guard error every parser raises, from its code, or `None` when each accepts the SQL. + pub(super) expected_error: Option, +} diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs index 5afda5f19d..22f36ac948 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/star_keys.rs @@ -8,6 +8,7 @@ use polyglot_sql::{ }; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; use crate::planner_fixtures::models::FixtureStarRelation; @@ -48,7 +49,7 @@ pub fn star_keys( tables, strict: None, }), - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), }; let Ok(sql) = stub_markers(sql) else { return Vec::new(); diff --git a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs index a84b3ee534..f6c6792626 100644 --- a/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs +++ b/crates/sqlbuild-analysis/src/planner_migrations/_helpers/proofs/parse.rs @@ -1,20 +1,15 @@ //! Parse migration SQL under the compiler's complexity limit. -use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; use polyglot_sql::dialects::Dialect; -use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; -use serde_json::{Value, json}; + +use serde_json::Value; + +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; pub(crate) fn parse(sql: &str, dialect: &str) -> Result, String> { let dialect = Dialect::get_by_name(dialect).ok_or_else(|| format!("unknown dialect {dialect}"))?; - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| error.to_string())?; - let options = ParseOptions { - complexity_guard: Some(guard), - }; - let Ok(mut statements) = dialect.parse_with_options(sql, &options) else { + let Ok(mut statements) = dialect.parse_with_options(sql, &polyglot_parse_options()) else { return Ok(None); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index cde00841a5..6bd5144f1e 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -15,6 +15,8 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; use crate::constants::{UNKNOWN_SQL_TYPE, VARCHAR_SQL_TYPE}; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use sqlbuild_core::constants::SQL_WILDCARD; const DEFAULT_WORKERS: usize = 4; @@ -89,6 +91,7 @@ impl AnalysisRequest { check_references: true, strict: Some(true), strict_syntax: false, + complexity_guard: Some(polyglot_complexity_guard()), ..Default::default() } } @@ -612,16 +615,9 @@ fn try_borrowed_query( if !matches!(dialect, DialectType::Snowflake | DialectType::DuckDB) { return Err(Box::new(work)); } - let Ok(mut statements) = polyglot_sql::parse_with_options( - &work.query.sql, - dialect, - &polyglot_sql::ParseOptions { - complexity_guard: Some(polyglot_sql::ComplexityGuardOptions { - max_function_call_depth: Some(128), - ..Default::default() - }), - }, - ) else { + let Ok(mut statements) = + polyglot_sql::parse_with_options(&work.query.sql, dialect, &polyglot_parse_options()) + else { return Err(Box::new(work)); }; if statements.len() != 1 { @@ -1036,7 +1032,7 @@ fn compile_query(mut request: AnalysisRequest, project_projections: bool) -> Com let compiled = polyglot_sql::compile_required_query_analysis( &request.sql, AnalyzeQueryOptions { - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), dialect, schema: request.schema, }, @@ -1105,7 +1101,7 @@ fn query_analysis( } }; let options = AnalyzeQueryOptions { - complexity_guard: None, + complexity_guard: Some(polyglot_complexity_guard()), dialect, schema: request.schema, }; diff --git a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs index 28fe14df79..ecd6d23d29 100644 --- a/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs +++ b/crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs @@ -1,12 +1,13 @@ //! Python's dict walk over each parsed statement: tables, CTE names and temporary tables. -use polyglot_sql::{ComplexityGuardOptions, Dialect, Expression, ParseOptions}; -use serde_json::{Map, Value, json}; +use polyglot_sql::{Dialect, Expression}; +use serde_json::{Map, Value}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::relation_names::constants::{ - CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, - MAX_FUNCTION_CALL_DEPTH, NAME_KEY, QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, + CATALOG_KEY, CREATE_TABLE_KEY, CTE_ALIAS_KEY, CTE_LIST_KEY, GENERIC_DIALECT, NAME_KEY, + QUALIFIER_SEPARATOR, SCHEMA_KEY, TABLE_KEYS, TEMPORARY_KEY, }; use crate::relation_names::errors::RelationNamesFailure; use crate::relation_names::models::{ExtractedRelations, RelationName}; @@ -53,13 +54,7 @@ fn parsed_statements( let parser: Dialect = Dialect::get_by_name(dialect).ok_or_else(|| { RelationNamesFailure::UnknownDialect(format!("Unknown dialect: {dialect}")) })?; - let guard: ComplexityGuardOptions = - serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH})) - .map_err(|error| RelationNamesFailure::Internal(error.to_string()))?; - let options: ParseOptions = ParseOptions { - complexity_guard: Some(guard), - }; - match parser.parse_with_options(sql, &options) { + match parser.parse_with_options(sql, &polyglot_parse_options()) { Ok(statements) => Ok(Some(statements)), Err(_polyglot_error) => Ok(None), } diff --git a/crates/sqlbuild-analysis/src/relation_names/constants.rs b/crates/sqlbuild-analysis/src/relation_names/constants.rs index f8cab78a94..1f93241a40 100644 --- a/crates/sqlbuild-analysis/src/relation_names/constants.rs +++ b/crates/sqlbuild-analysis/src/relation_names/constants.rs @@ -11,5 +11,3 @@ pub(crate) const CREATE_TABLE_KEY: &str = "create_table"; pub(crate) const TEMPORARY_KEY: &str = "temporary"; pub(crate) const QUALIFIER_SEPARATOR: char = '.'; pub(crate) const GENERIC_DIALECT: &str = "generic"; -/// The guarded proxy's `maxFunctionCallDepth` for the wheel's `parse`. -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; diff --git a/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs index f09b0f5f66..5931fd33ff 100644 --- a/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs +++ b/crates/sqlbuild-analysis/src/required_columns/main/resolve_required_columns.rs @@ -1,9 +1,10 @@ use std::collections::HashMap; use std::rc::Rc; -use polyglot_sql::{Dialect, Expression, ParseOptions}; +use polyglot_sql::{Dialect, Expression}; use sqlbuild_core::text::main::python_casefold::python_casefold; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::required_columns::_helpers::queries::query_reads; use crate::required_columns::_helpers::relations::{RelationMap, RelationReads, unique_reads}; use crate::required_columns::constants::{GENERIC_DIALECT, TABLE_FUNCTION_STUB_PREFIX}; @@ -28,7 +29,7 @@ pub fn resolve_required_columns( .map_err(RequiredColumnsError::Normalization)?; let parser: Dialect = Dialect::get_by_name(dialect_name) .ok_or_else(|| RequiredColumnsError::UnknownDialect(dialect_name.to_owned()))?; - let Ok(mut statements) = parser.parse_with_options(&sql, &ParseOptions::default()) else { + let Ok(mut statements) = parser.parse_with_options(&sql, &polyglot_parse_options()) else { return Ok(Vec::new()); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs index ca3f08dca7..9f240f0365 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/sql_text/parsed_sql.rs @@ -3,13 +3,11 @@ use std::collections::{HashMap, HashSet}; use polyglot_sql::tokens::TokenType; -use polyglot_sql::{ - ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, ParseOptions, -}; +use polyglot_sql::{Dialect, DialectType, Expression, ExpressionWalk}; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_checks::constants::{ - CLOSE_PARENTHESIS, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, OPEN_PARENTHESIS, - PROJECTION_SEPARATOR, + CLOSE_PARENTHESIS, GENERIC_DIALECT, OPEN_PARENTHESIS, PROJECTION_SEPARATOR, }; use crate::semantic_checks::models::SemanticFailure; use crate::semantic_validation::main::normalize::normalize_analysis_sql; @@ -120,14 +118,7 @@ pub(crate) fn parsed_model( ) -> Result, SemanticFailure> { let sql = normalized_sql(query_sql, dialect)?; let parser = polyglot_dialect(dialect.unwrap_or(GENERIC_DIALECT))?; - let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ - "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, - })) - .map_err(|_| SemanticFailure::internal("the parse guard options do not build"))?; - let options = ParseOptions { - complexity_guard: Some(guard), - }; - let Ok(mut statements) = parser.parse_with_options(&sql, &options) else { + let Ok(mut statements) = parser.parse_with_options(&sql, &polyglot_parse_options()) else { return Ok(None); }; if statements.len() != 1 { diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 98800e3e4d..0b0b44e949 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -5,7 +5,6 @@ pub(crate) const COMPARISON_CODE: &str = "B217"; pub(crate) const SQL_TEST_COLUMN_CODE: &str = "B302"; pub(crate) const SEMANTIC_CODE_PREFIX: &str = "B"; pub(crate) const MODEL_RESOURCE_TYPE: &str = "model"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: u64 = 512; pub(crate) const GENERIC_DIALECT: &str = "generic"; pub(crate) const OPEN_PARENTHESIS: &str = "("; pub(crate) const CLOSE_PARENTHESIS: &str = ")"; diff --git a/crates/sqlbuild-analysis/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs index beeedb31e0..623070bd77 100644 --- a/crates/sqlbuild-analysis/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -1,5 +1,7 @@ //! On-demand direct non-projection column usage extraction. +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, Resolver, SchemaValidationOptions, ValidationSchema, build_scope, mapping_schema_from_validation_schema_with_dialect, @@ -62,6 +64,7 @@ pub fn analyze_json(request_json: &str) -> Result { &SchemaValidationOptions { check_references: true, strict: Some(true), + complexity_guard: Some(polyglot_complexity_guard()), ..SchemaValidationOptions::default() }, ); @@ -70,7 +73,7 @@ pub fn analyze_json(request_json: &str) -> Result { .map_err(|error| error.to_string()); } let statements = Dialect::get(dialect) - .parse(&request.sql) + .parse_with_options(&request.sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; let schema = mapping_schema_from_validation_schema_with_dialect(&request.schema, dialect); let mut uses: Vec = Vec::new(); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index 4a5dd340db..5262ddcd16 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -1,5 +1,7 @@ //! Compile-owned schema catalog and native, batched binding requests. +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::main::normalize_batch; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -68,7 +70,7 @@ impl ProjectCatalog { ) -> Result)>, String> { use polyglot_sql::{Expression, ExpressionWalk}; let parsed = Dialect::get(self.dialect) - .parse(sql) + .parse_with_options(sql, &polyglot_parse_options()) .map_err(|error| error.to_string())?; let mut identifiers: HashMap = HashMap::new(); for expression in &parsed { @@ -151,10 +153,7 @@ impl ProjectCatalog { check_references: true, semantic: true, strict: Some(true), - complexity_guard: Some(polyglot_sql::ComplexityGuardOptions { - max_function_call_depth: Some(512), - ..Default::default() - }), + complexity_guard: Some(polyglot_complexity_guard()), ..Default::default() }, tables: HashMap::new(), diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index a60d49f7ab..4778cb130a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -1,5 +1,6 @@ //! SQLBuild codes and dialect-specific diagnostic evidence. +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; use crate::semantic_validation::_helpers::sqlite_temporal_casts; @@ -76,8 +77,9 @@ struct DiagnosticContext<'a> { impl DiagnosticContext<'_> { fn parsed(&mut self) -> &polyglot_sql::Result> { - self.statements - .get_or_insert_with(|| Dialect::get(self.dialect).parse(self.sql)) + self.statements.get_or_insert_with(|| { + Dialect::get(self.dialect).parse_with_options(self.sql, &polyglot_parse_options()) + }) } fn map(&mut self, mut error: ValidationError) -> Result, String> { diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index fb9ca0d11e..bd9e7446b5 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -14,6 +14,8 @@ pub mod normalize_native_renamed; pub mod sqlbuild_code; pub mod text_findings; pub mod truth_test_spans; +use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::models::ProjectCatalog; use polyglot_sql::{ @@ -88,6 +90,16 @@ pub(super) fn validation_result(request: ValidationRequest) -> Result bool { @@ -18,11 +19,8 @@ pub(crate) fn python_space(character: char) -> bool { /// Whether a CREATE statement's text parses as a definition that needs no warehouse. pub(crate) fn metadata_only_create(query_text: &str) -> bool { - let options: ParseOptions = ParseOptions { - complexity_guard: None, - }; - let Ok(mut statements) = - Dialect::get(DialectType::Snowflake).parse_with_options(query_text, &options) + let Ok(mut statements) = Dialect::get(DialectType::Snowflake) + .parse_with_options(query_text, &polyglot_parse_options()) else { return false; }; diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests.rs b/crates/sqlbuild-analysis/src/sql_macros/tests.rs index 1dc7f7981e..4af625b2e9 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_call_depth.rs"] +mod test_call_depth; #[path = "tests/test_definitions.rs"] mod test_definitions; #[path = "tests/test_expansion.rs"] diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs index 105be1d1f6..5ccf28eae7 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/helpers.rs @@ -233,3 +233,18 @@ pub(super) fn raw_prefix_rules() -> sqlbuild_sqltext::sql_scan::models::LexicalS ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() } } + +/// An expression macro whose body nests `depth` `REPLACE` calls around its argument. +pub(super) fn nested_replace_macro(depth: usize) -> String { + let mut source: String = String::from( + "MACRO (description \"Cleaned name\", arguments (column_name VARCHAR), returns VARCHAR);\n\n", + ); + let closings: String = (0..depth) + .map(|level| format!(", 'a{level}', 'b')")) + .collect(); + source.push_str(&format!( + "{}@column_name{closings}\n", + "REPLACE(".repeat(depth) + )); + source +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs new file mode 100644 index 0000000000..f063015d43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs @@ -0,0 +1,45 @@ +use crate::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::sql_macros::tests::helpers::{expanded_for, nested_replace_macro}; +use crate::sql_macros::tests::test_types::NestedMacroTestCase; + +#[test] +fn given_a_deeply_nested_macro_body_when_expanding_for_snowflake_then_the_shared_limit_applies() { + let test_cases = [ + NestedMacroTestCase { + description: "a hundred nested calls expand", + depth: 100, + expected: Ok(100), + }, + NestedMacroTestCase { + description: "one past the limit is P015 naming the limit", + depth: MAX_FUNCTION_CALL_DEPTH + 1, + expected: Err(format!( + "body of macro cleaned_name() is not a valid expression macro: \ + E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value {} exceeds configured limit \ + {MAX_FUNCTION_CALL_DEPTH}", + MAX_FUNCTION_CALL_DEPTH + 1 + )), + }, + ]; + + for test_case in test_cases { + let source: String = nested_replace_macro(test_case.depth); + + let outcome = expanded_for( + "snowflake", + &[("cleaned_name", &source)], + "SELECT @sql.cleaned_name(race_name) AS cleaned FROM races", + ) + .map(|sql| sql.matches("REPLACE(").count()) + .map_err(|error| format!("{}: {}", error.code, error.message)); + + assert_eq!( + outcome, + test_case + .expected + .map_err(|message| format!("P015: {message}")), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 26fdebc00a..8c72b7202f 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -135,3 +135,10 @@ pub(crate) struct ProbeSchemaLookupTestCase { pub(crate) relation: &'static str, pub(crate) expected_column: Option<&'static str>, } + +pub(super) struct NestedMacroTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + /// The expansion's `REPLACE` count, or the P015 message text when the body is too deep. + pub(super) expected: Result, +} diff --git a/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs b/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs index 705b3a0834..f9adb36c2e 100644 --- a/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs +++ b/crates/sqlbuild-analysis/src/sql_tree/_helpers/parse.rs @@ -1,7 +1,8 @@ //! Parse SQL into statements with node spans. -use sqlquack_syntax::{CallMode, Expression, ParseOptions}; +use sqlquack_syntax::{CallMode, Expression}; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::sql_tree::constants::GUARD_CODE_PREFIX; use crate::sql_tree::errors::SqlTreeError; use crate::sql_tree::models::TreeParser; @@ -12,7 +13,7 @@ pub(crate) fn parse(sql: &str, parser: TreeParser) -> Result, Sq TreeParser::Shared(dialect) => sqlquack_parse::parse_statements_with( sql, dialect, - &ParseOptions::default(), + &sqlquack_parse_options(), CallMode::FaithfulLocated, ) .map_err(|error| shared_error(&error.to_string())), diff --git a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs index b709dc1b9f..fe8e746a20 100644 --- a/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs +++ b/crates/sqlbuild-analysis/src/warehouse_types/_helpers/type_trees.rs @@ -2,9 +2,10 @@ use serde_json::Value; use sqlquack_parse::parse_strict; +use sqlquack_syntax::Expression; use sqlquack_syntax::tokens::models::Dialect; -use sqlquack_syntax::{Expression, ParseOptions}; +use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use crate::warehouse_types::constants::{CAST_KEY, CAST_TARGET_KEY}; /// `spelling` read by `sqlquack-syntax-postgres`'s type grammar. @@ -17,7 +18,7 @@ pub(crate) fn postgres_type(spelling: &str) -> Result { /// `spelling` read as a cast target by SQLQuack's core parser for `dialect`. pub(crate) fn core_type(spelling: &str, dialect: Dialect) -> Result { let statements: Vec = - parse_strict(&cast(spelling), dialect, &ParseOptions::default()) + parse_strict(&cast(spelling), dialect, &sqlquack_parse_options()) .map_err(|error| format!("{error:?}"))?; single_cast_target(spelling, &statements) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 510cd06cc0..108c373165 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -8,6 +8,7 @@ use sqlbuild_analysis::fingerprint_storage::constants::{ COLUMN_TARGET_DATABASE, COLUMN_TARGET_NAME, COLUMN_TARGET_SCHEMA, COLUMN_TIMESTAMP, COLUMN_VERSION_HASH, NODE_TYPE_MODEL, NODE_TYPE_SEED, }; +use sqlbuild_analysis::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; use sqlbuild_analysis::snowflake_cost::constants::QUERY_TAG_APP; use sqlbuild_cli::compute_logs::constants::{ COMPUTE_LOG_FORMAT_VERSION, DEFAULT_RETENTION_COUNT, MAX_READ_BYTES, @@ -141,6 +142,7 @@ pub(crate) const EXPORTED_INTEGERS: &[(&str, i64)] = &[ "CUSTOM_HOST_PROTOCOL_VERSION", CUSTOM_HOST_PROTOCOL_VERSION as i64, ), + ("MAX_FUNCTION_CALL_DEPTH", MAX_FUNCTION_CALL_DEPTH as i64), ]; /// Text tuples exported under their Python names. pub(crate) const EXPORTED_TEXT_TUPLES: &[(&str, &[&str])] = &[( diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs index 02fcf32612..44c326018b 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::LazyLock; use regex::Regex; +use sqlbuild_analysis::parse_guard::constants::MAX_FUNCTION_CALL_DEPTH; use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_core::text::main::python_isspace::python_isspace; @@ -21,8 +22,8 @@ use crate::lint_text::constants::{ BLOCK_COMMENT_OPENER, DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, EXPRESSION_SELECT_PREFIX, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, FUNCTION_HEADER_KIND, GLOBAL_NAMED_ROOTS, GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, LINTED_EXPRESSION_ALIAS, - MACRO_HEADER_KIND, MAX_FUNCTION_CALL_DEPTH, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, - OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, + MACRO_HEADER_KIND, MOVED_EXPRESSION_ALIAS, MOVED_OMITTED_SELECT, OMITTED_CEREMONIAL_SELECT_SQL, + OMITTED_SELECT_SUFFIX_PATTERN, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index e2dac213d3..a5188d2bf4 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -67,7 +67,6 @@ pub(crate) const UNIT_TEST_ROOT: [&str; 2] = ["tests", "unit"]; pub(crate) const SQL_SUFFIX: &str = ".sql"; pub(crate) const FORMAT_API_VERSION: u32 = 1; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; pub(crate) const OMITTED_CEREMONIAL_SELECT_SQL: &str = "\nSELECT 1"; pub(crate) const OMITTED_SELECT_SUFFIX_PATTERN: &str = r"(?i)\s*\bSELECT\s+1\s*\z"; pub(crate) const FIXTURE_HEADER_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs index 46ca4fa8af..ca5e1ce889 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/facts.rs @@ -2,20 +2,18 @@ use crate::query_facts::_helpers::flatten::{flatten_call, flatten_input, whole_v use crate::query_facts::_helpers::shape::stub_call; use crate::query_facts::_helpers::tokens::{ComparisonOperator, QueryTokens, SelectTokens}; use crate::query_facts::constants::{ - MAX_AST_DEPTH, MAX_FUNCTION_CALL_DEPTH, SNOWFLAKE_DATE_PART_ARGUMENTS, TABLE_FUNCTION_STUB, - TRIPLE_QUOTE_LENGTH, + MAX_AST_DEPTH, SNOWFLAKE_DATE_PART_ARGUMENTS, TABLE_FUNCTION_STUB, TRIPLE_QUOTE_LENGTH, }; use crate::query_facts::models::{ CallKind, ColumnFact, ComparisonFacts, ComparisonSpan, CteFacts, ExpressionItem, ItemFacts, ModelShape, ParsedFixture, QueryFacts, RelationFacts, SelectFacts, SetFacts, SourceText, }; +use sqlbuild_analysis::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options; use sqlbuild_analysis::sql_tree::main::deeper_than::deeper_than; use sqlbuild_analysis::sql_tree::main::shared_dialect::shared_dialect; use sqlquack_syntax::expressions::names::models::{Identifier, Literal, TableRef}; use sqlquack_syntax::expressions::queries::models::{Select, With}; -use sqlquack_syntax::{ - CallMode, ComplexityGuardOptions, Dialect, Expression, ExpressionWalk, ParseOptions, -}; +use sqlquack_syntax::{CallMode, Dialect, Expression, ExpressionWalk}; use std::cell::{Cell, RefCell}; use std::collections::{HashMap, HashSet}; use std::ops::Range; @@ -23,12 +21,7 @@ use std::ops::Range; fn parse_query(sql: &str, dialect_name: &str) -> Result<(Expression, Dialect), String> { let dialect = shared_dialect(dialect_name) .ok_or_else(|| format!("no SQL parser for dialect {dialect_name}"))?; - let options = ParseOptions { - complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), - ..Default::default() - }), - }; + let options = sqlquack_parse_options(); let mut statements = sqlquack_parse::parse_statements_with(sql, dialect, &options, CallMode::FaithfulLocated) .map_err(|error| error.to_string())?; diff --git a/crates/sqlbuild-rules/src/query_facts/constants.rs b/crates/sqlbuild-rules/src/query_facts/constants.rs index ae3601132a..df202a3229 100644 --- a/crates/sqlbuild-rules/src/query_facts/constants.rs +++ b/crates/sqlbuild-rules/src/query_facts/constants.rs @@ -5,7 +5,6 @@ pub(crate) const TABLE_FUNCTION_STUB: &str = "__sqlbuild_table_function_"; pub(crate) const UDF_STUB: &str = "__sqlbuild_udf_"; pub(crate) const PLACEHOLDER_PREFIX: &[u8] = b"@@@"; pub(crate) const PLACEHOLDER_VALUE: &str = "NULL"; -pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 128; /// The deepest statement built-in rules read, Polyglot's former AST-depth limit. pub(crate) const MAX_AST_DEPTH: usize = 512; pub(crate) const UDF_CALL: &str = "__udf"; diff --git a/scripts/check_parse_guard.py b/scripts/check_parse_guard.py new file mode 100644 index 0000000000..6539283c39 --- /dev/null +++ b/scripts/check_parse_guard.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the shared parse guard check.""" + +from __future__ import annotations + +from scripts.parse_guard.main.check import run_parse_guard_check + + +def main(argv: list[str] | None = None) -> int: + """Check that native SQL parses use the shared parse guard.""" + return run_parse_guard_check(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/parse_guard/__init__.py b/scripts/parse_guard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/_helpers/__init__.py b/scripts/parse_guard/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/_helpers/scan.py b/scripts/parse_guard/_helpers/scan.py new file mode 100644 index 0000000000..4687f2d2f2 --- /dev/null +++ b/scripts/parse_guard/_helpers/scan.py @@ -0,0 +1,85 @@ +"""Find native SQL parses that fall back to a library's default function-call depth.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from scripts.parse_guard.constants import ( + CRATES_DIRECTORY, + DEFAULT_GUARD_PATTERNS, + GUARD_FIELD, + GUARDED_OPTION_LITERALS, + SKIPPED_DIRECTORIES, + SKIPPED_FILES, + SOURCE_SUFFIX, + TYPE_POSITION_SUFFIXES, +) + +_PATTERNS: tuple[re.Pattern[str], ...] = tuple( + re.compile(pattern, re.DOTALL) for pattern in DEFAULT_GUARD_PATTERNS +) + + +def crate_sources(*, root: Path) -> tuple[Path, ...]: + """Every non-test Rust source under the workspace crates, except the shared guard.""" + sources: list[Path] = [] + path: Path + for path in (root / CRATES_DIRECTORY).rglob(f"*{SOURCE_SUFFIX}"): + skipped: bool = bool(SKIPPED_DIRECTORIES.intersection(path.relative_to(root).parts)) + if path.name not in SKIPPED_FILES and not skipped: + sources.append(path) + return tuple(sorted(sources)) + + +def unguarded_sites(*, label: str, text: str) -> tuple[str, ...]: + """Each place in `text` a parse uses a default guard, as `label:line pattern`.""" + sites: list[str] = [] + pattern: re.Pattern[str] + for pattern in _PATTERNS: + sites.extend( + f"{label}:{_line(text=text, offset=found.start())} {pattern.pattern}" + for found in pattern.finditer(text) + ) + literal: str + for literal in GUARDED_OPTION_LITERALS: + sites.extend( + f"{label}:{_line(text=text, offset=found.start())} {literal}" + for found in re.finditer(re.escape(literal), text) + if _unguarded_literal(text=text, start=found.start(), literal=literal) + ) + return tuple(sites) + + +def get_parse_guard_errors(root: Path) -> tuple[str, ...]: + """Every unguarded parse site of the workspace crates.""" + errors: list[str] = [] + path: Path + for path in crate_sources(root=root): + errors.extend( + unguarded_sites( + label=path.relative_to(root).as_posix(), text=path.read_text(encoding="utf-8") + ) + ) + return tuple(errors) + + +def _line(*, text: str, offset: int) -> int: + return text.count("\n", 0, offset) + 1 + + +def _unguarded_literal(*, text: str, start: int, literal: str) -> bool: + if text[:start].rstrip().endswith(TYPE_POSITION_SUFFIXES): + return False + return GUARD_FIELD not in _block(text=text, open_at=start + len(literal) - 1) + + +def _block(*, text: str, open_at: int) -> str: + depth: int = 0 + offset: int + character: str + for offset, character in enumerate(text[open_at:]): + depth += {"{": 1, "}": -1}.get(character, 0) + if depth == 0: + return text[open_at : open_at + offset + 1] + return text[open_at:] diff --git a/scripts/parse_guard/constants.py b/scripts/parse_guard/constants.py new file mode 100644 index 0000000000..0581a35178 --- /dev/null +++ b/scripts/parse_guard/constants.py @@ -0,0 +1,22 @@ +"""Text that marks a native SQL parse using a library's default complexity guard.""" + +CRATES_DIRECTORY: str = "crates" +SOURCE_SUFFIX: str = ".rs" +SKIPPED_DIRECTORIES: frozenset[str] = frozenset({"tests", "target", "parse_guard"}) +SKIPPED_FILES: frozenset[str] = frozenset({"tests.rs"}) +DEFAULT_GUARD_PATTERNS: tuple[str, ...] = ( + r"ParseOptions::default\(\)", + r"complexity_guard: None", + r"maxFunctionCallDepth", + r"max_function_call_depth: Some\((?!MAX_FUNCTION_CALL_DEPTH\))", + r"const MAX_FUNCTION_CALL_DEPTH", + r"polyglot_sql::(?:parse|transpile|validate)\(", + r"Dialect::get(?:_by_name)?\([^;]*?\)\s*(?:\.[a-z_]+\([^;]*?\)\??\s*)*?\.parse\(", +) +GUARDED_OPTION_LITERALS: tuple[str, ...] = ( + "SchemaValidationOptions {", + "AnalyzeQueryOptions {", + "ValidationOptions {", +) +GUARD_FIELD: str = "complexity_guard" +TYPE_POSITION_SUFFIXES: tuple[str, ...] = ("->", "struct") diff --git a/scripts/parse_guard/main/__init__.py b/scripts/parse_guard/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/parse_guard/main/check.py b/scripts/parse_guard/main/check.py new file mode 100644 index 0000000000..8fca257701 --- /dev/null +++ b/scripts/parse_guard/main/check.py @@ -0,0 +1,27 @@ +"""Parse guard command entrypoint.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from scripts.parse_guard._helpers.scan import get_parse_guard_errors + + +def run_parse_guard_check(argv: list[str] | None = None) -> int: + """Fail when a native SQL parse uses a library's default function-call depth.""" + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + arguments: argparse.Namespace = parser.parse_args(argv) + errors: tuple[str, ...] = get_parse_guard_errors(arguments.root) + if errors: + print( + "Parse with sqlbuild_analysis::parse_guard's options, not a library default:", + file=sys.stderr, + ) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Every native SQL parse uses the shared parse guard.") + return 0 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 47b8971951..2626a77c8d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -227,6 +227,7 @@ CURRENT_EVENT_LOG_SCHEMA_VERSION: int CURRENT_RUN_STORAGE_SCHEMA_VERSION: int COMPUTE_LOG_FORMAT_VERSION: int DEFAULT_RETENTION_COUNT: int +MAX_FUNCTION_CALL_DEPTH: int MAX_READ_BYTES: int CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int MIGRATION_WRITE_ATTEMPTS: int diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index d68a9dc000..6f78551182 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -2,10 +2,12 @@ from __future__ import annotations +from sqlbuild._native import MAX_FUNCTION_CALL_DEPTH + POLYGLOT_ANALYZE_QUERY_ENTRY_POINT: str = "analyze_query" POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT: int = 2 POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX: int = 1 POLYGLOT_COMPLEXITY_GUARD_PARAMETER: str = "complexity_guard" POLYGLOT_COMPLEXITY_GUARD_OPTION: str = "complexityGuard" POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION: str = "maxFunctionCallDepth" -POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 +POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = MAX_FUNCTION_CALL_DEPTH diff --git a/tests/unit/scripts/parse_guard/__init__.py b/tests/unit/scripts/parse_guard/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/parse_guard/_helpers/__init__.py b/tests/unit/scripts/parse_guard/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/parse_guard/_helpers/_test_types.py b/tests/unit/scripts/parse_guard/_helpers/_test_types.py new file mode 100644 index 0000000000..14a0aafde2 --- /dev/null +++ b/tests/unit/scripts/parse_guard/_helpers/_test_types.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class UnguardedSiteTestCase: + description: str + text: str + expected_sites: tuple[str, ...] + + +@dataclass(frozen=True) +class RepositoryParseGuardTestCase: + description: str + expected_errors: tuple[str, ...] diff --git a/tests/unit/scripts/parse_guard/_helpers/test_scan.py b/tests/unit/scripts/parse_guard/_helpers/test_scan.py new file mode 100644 index 0000000000..eed3e400e4 --- /dev/null +++ b/tests/unit/scripts/parse_guard/_helpers/test_scan.py @@ -0,0 +1,73 @@ +from pathlib import Path + +import pytest + +from scripts.parse_guard._helpers.scan import get_parse_guard_errors, unguarded_sites +from tests.unit.scripts.parse_guard._helpers._test_types import ( + RepositoryParseGuardTestCase, + UnguardedSiteTestCase, +) + +_REPOSITORY_ROOT: Path = Path(__file__).resolve().parents[5] + + +@pytest.mark.parametrize( + "test_case", + [ + UnguardedSiteTestCase( + description="a bare Polyglot parse across lines", + text="fn a() {\n Dialect::get(d)\n .parse(sql)\n}\n", + expected_sites=( + r"x.rs:2 Dialect::get(?:_by_name)?\([^;]*?\)\s*(?:\.[a-z_]+\([^;]*?\)\??\s*)*?" + r"\.parse\(", + ), + ), + UnguardedSiteTestCase( + description="SQLQuack's default options", + text="fn b() { parse_strict(sql, d, &ParseOptions::default()); }", + expected_sites=(r"x.rs:1 ParseOptions::default\(\)",), + ), + UnguardedSiteTestCase( + description="a local depth instead of the shared constant", + text="fn c() { Guard { max_function_call_depth: Some(512) } }", + expected_sites=( + r"x.rs:1 max_function_call_depth: Some\((?!MAX_FUNCTION_CALL_DEPTH\))", + ), + ), + UnguardedSiteTestCase( + description="an analysis options literal without a guard", + text="fn d() { let o = AnalyzeQueryOptions { dialect, schema: Some(S { t }) }; }", + expected_sites=("x.rs:1 AnalyzeQueryOptions {",), + ), + UnguardedSiteTestCase( + description="the shared options and constant", + text=( + "fn e() -> AnalyzeQueryOptions { AnalyzeQueryOptions { complexity_guard: g } }\n" + "fn f() { Dialect::get(d).parse_with_options(sql, &polyglot_parse_options()); }\n" + "fn g() { R { max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH) } }\n" + ), + expected_sites=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_parse_text_when_scanning_then_reports_each_default_guarded_site( + test_case: UnguardedSiteTestCase, +) -> None: + assert unguarded_sites(label="x.rs", text=test_case.text) == test_case.expected_sites + + +@pytest.mark.parametrize( + "test_case", + [ + RepositoryParseGuardTestCase( + description="every workspace parse uses the shared guard", + expected_errors=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_repository_when_checking_parse_guard_then_returns_expected_errors( + test_case: RepositoryParseGuardTestCase, +) -> None: + assert get_parse_guard_errors(_REPOSITORY_ROOT) == test_case.expected_errors From 5e98608a9428e41a5f38b3adcc4879f5933a510c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:52:05 +0000 Subject: [PATCH 2/5] test(analysis): name the nested macro case's expectation expected_outcome Refs: fensu FPRST202/FPRST404 --- .../src/sql_macros/tests/test_call_depth.rs | 6 +++--- crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs index f063015d43..0df580118c 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_call_depth.rs @@ -8,12 +8,12 @@ fn given_a_deeply_nested_macro_body_when_expanding_for_snowflake_then_the_shared NestedMacroTestCase { description: "a hundred nested calls expand", depth: 100, - expected: Ok(100), + expected_outcome: Ok(100), }, NestedMacroTestCase { description: "one past the limit is P015 naming the limit", depth: MAX_FUNCTION_CALL_DEPTH + 1, - expected: Err(format!( + expected_outcome: Err(format!( "body of macro cleaned_name() is not a valid expression macro: \ E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value {} exceeds configured limit \ {MAX_FUNCTION_CALL_DEPTH}", @@ -36,7 +36,7 @@ fn given_a_deeply_nested_macro_body_when_expanding_for_snowflake_then_the_shared assert_eq!( outcome, test_case - .expected + .expected_outcome .map_err(|message| format!("P015: {message}")), "{}", test_case.description diff --git a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs index 8c72b7202f..a8a3668922 100644 --- a/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/sql_macros/tests/test_types.rs @@ -140,5 +140,5 @@ pub(super) struct NestedMacroTestCase { pub(super) description: &'static str, pub(super) depth: usize, /// The expansion's `REPLACE` count, or the P015 message text when the body is too deep. - pub(super) expected: Result, + pub(super) expected_outcome: Result, } From 5b3f7a0884b615007f28d43f2ec2338584f7dbfd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:40:49 +0000 Subject: [PATCH 3/5] refactor(scripts): run the root checks through one shared runner The parse guard, native stub, native layering and release gate checks each parsed --root, listed their errors under a heading and printed a success line in the same way. They now share run_root_check and keep only what they check and the messages they print. --- scripts/native_layering/main/check.py | 26 +++++++++----------- scripts/native_stub/main/check.py | 29 +++++++++++----------- scripts/parse_guard/main/check.py | 26 +++++++------------- scripts/release_gate/main/check.py | 29 +++++++++++----------- scripts/root_check/__init__.py | 0 scripts/root_check/main/__init__.py | 0 scripts/root_check/main/run_root_check.py | 30 +++++++++++++++++++++++ 7 files changed, 80 insertions(+), 60 deletions(-) create mode 100644 scripts/root_check/__init__.py create mode 100644 scripts/root_check/main/__init__.py create mode 100644 scripts/root_check/main/run_root_check.py diff --git a/scripts/native_layering/main/check.py b/scripts/native_layering/main/check.py index 4c4e6a2711..1db7c6318c 100644 --- a/scripts/native_layering/main/check.py +++ b/scripts/native_layering/main/check.py @@ -2,26 +2,22 @@ from __future__ import annotations -import argparse -import sys from pathlib import Path from scripts.native_layering._helpers.layering import get_native_layering_errors from scripts.native_layering._helpers.versions import get_native_version_errors +from scripts.root_check.main.run_root_check import run_root_check def run_native_layering_check(argv: list[str] | None = None) -> int: """Fail when a native crate breaks the declared layering or the release versions differ.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) - errors: tuple[str, ...] = get_native_layering_errors( - arguments.root - ) + get_native_version_errors(arguments.root) - if errors: - print("Native crate layering is invalid:", file=sys.stderr) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Native crate layering is valid.") - return 0 + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_layering_and_version_errors, + messages=("Native crate layering is invalid:", "Native crate layering is valid."), + ) + + +def _layering_and_version_errors(root: Path) -> tuple[str, ...]: + return get_native_layering_errors(root) + get_native_version_errors(root) diff --git a/scripts/native_stub/main/check.py b/scripts/native_stub/main/check.py index 17af5ff84c..3d14757250 100644 --- a/scripts/native_stub/main/check.py +++ b/scripts/native_stub/main/check.py @@ -2,27 +2,28 @@ from __future__ import annotations -import argparse import importlib -import sys from pathlib import Path from scripts.native_stub._helpers.declarations import stub_drift from scripts.native_stub.constants import NATIVE_MODULE, STUB_PATH +from scripts.root_check.main.run_root_check import run_root_check def run_native_stub_check(argv: list[str] | None = None) -> int: """Fail when the stub declares a runtime name the built native extension does not register.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_unregistered_names, + messages=( + f"{STUB_PATH} declares names {NATIVE_MODULE} does not register:", + f"{STUB_PATH} declares only registered names.", + ), + ) + + +def _unregistered_names(root: Path) -> tuple[str, ...]: registered: frozenset[str] = frozenset(dir(importlib.import_module(NATIVE_MODULE))) - stub_text: str = (arguments.root / STUB_PATH).read_text(encoding="utf-8") - missing: tuple[str, ...] = stub_drift(stub_text=stub_text, registered=registered) - if missing: - print(f"{STUB_PATH} declares names {NATIVE_MODULE} does not register:", file=sys.stderr) - for name in missing: - print(f"- {name}", file=sys.stderr) - return 1 - print(f"{STUB_PATH} declares only registered names.") - return 0 + stub_text: str = (root / STUB_PATH).read_text(encoding="utf-8") + return stub_drift(stub_text=stub_text, registered=registered) diff --git a/scripts/parse_guard/main/check.py b/scripts/parse_guard/main/check.py index 8fca257701..9566a37aaf 100644 --- a/scripts/parse_guard/main/check.py +++ b/scripts/parse_guard/main/check.py @@ -2,26 +2,18 @@ from __future__ import annotations -import argparse -import sys -from pathlib import Path - from scripts.parse_guard._helpers.scan import get_parse_guard_errors +from scripts.root_check.main.run_root_check import run_root_check def run_parse_guard_check(argv: list[str] | None = None) -> int: """Fail when a native SQL parse uses a library's default function-call depth.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) - errors: tuple[str, ...] = get_parse_guard_errors(arguments.root) - if errors: - print( + return run_root_check( + argv=argv, + description=__doc__, + errors_under=get_parse_guard_errors, + messages=( "Parse with sqlbuild_analysis::parse_guard's options, not a library default:", - file=sys.stderr, - ) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Every native SQL parse uses the shared parse guard.") - return 0 + "Every native SQL parse uses the shared parse guard.", + ), + ) diff --git a/scripts/release_gate/main/check.py b/scripts/release_gate/main/check.py index 11194a2fb7..6d7c916a82 100644 --- a/scripts/release_gate/main/check.py +++ b/scripts/release_gate/main/check.py @@ -2,27 +2,28 @@ from __future__ import annotations -import argparse -import sys from pathlib import Path from scripts.release_gate._helpers.gate import get_release_gate_errors from scripts.release_gate.exceptions import ReleaseGateError +from scripts.root_check.main.run_root_check import run_root_check def run_release_gate_check(argv: list[str] | None = None) -> int: """Fail when anything but the trusted release checks can pass a release pull request.""" - parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path.cwd()) - arguments: argparse.Namespace = parser.parse_args(argv) + return run_root_check( + argv=argv, + description=__doc__, + errors_under=_gate_errors, + messages=( + "Release pull request Verify gate is invalid:", + "Release pull request Verify gate is valid.", + ), + ) + + +def _gate_errors(root: Path) -> tuple[str, ...]: try: - errors: tuple[str, ...] = get_release_gate_errors(arguments.root) + return get_release_gate_errors(root) except ReleaseGateError as error: - errors = (str(error),) - if errors: - print("Release pull request Verify gate is invalid:", file=sys.stderr) - for error in errors: - print(f"- {error}", file=sys.stderr) - return 1 - print("Release pull request Verify gate is valid.") - return 0 + return (str(error),) diff --git a/scripts/root_check/__init__.py b/scripts/root_check/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/root_check/main/__init__.py b/scripts/root_check/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/root_check/main/run_root_check.py b/scripts/root_check/main/run_root_check.py new file mode 100644 index 0000000000..203b68c17a --- /dev/null +++ b/scripts/root_check/main/run_root_check.py @@ -0,0 +1,30 @@ +"""Run one repository check over `--root` and report its errors.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Callable +from pathlib import Path + + +def run_root_check( + *, + argv: list[str] | None, + description: str | None, + errors_under: Callable[[Path], tuple[str, ...]], + messages: tuple[str, str], +) -> int: + """Print the check's errors under the failure heading and fail, or print the success line.""" + failure_heading, success_line = messages + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=description) + parser.add_argument("--root", type=Path, default=Path.cwd()) + arguments: argparse.Namespace = parser.parse_args(argv) + errors: tuple[str, ...] = errors_under(arguments.root) + if errors: + print(failure_heading, file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print(success_line) + return 0 From 3c8e750bc4def8ab8aaf55b020256252b8ac59c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:50:09 +0000 Subject: [PATCH 4/5] fix(analysis): name the parsers the shared call depth covers and close its gaps The shared 256 function-call depth covers every Polyglot parse and SQLQuack's shared and Snowflake parsers. SQLQuack's per-dialect parser crates (BigQuery, ClickHouse, Databricks, DuckDB, PostgreSQL, SQLite and T-SQL) take no options and keep their own built-in limits, so they are a documented exclusion until SQLQuack accepts options for them. The module docs and the check's messages now say so instead of every SQL parse. - Semantic validation always applies the shared guard; a request's own guard no longer overrides it. - Query enrichment sets the typed guard on its analysis options instead of round-tripping it through JSON. - The guarded_parse_options and proxy_parse_options pass-throughs, which could not fail, are gone; callers use polyglot_parse_options directly. - The parse-guard check also flags a ParseOptions literal without a guard and a parse_data_type call without options. --- .../analysis_session/_helpers/enrichment.rs | 7 ++---- .../_helpers/polyglot/function_types.rs | 5 ++-- .../src/compiler/_helpers/polyglot/mod.rs | 1 - .../_helpers/polyglot/parse_options.rs | 9 -------- .../src/lineage/_helpers/parsed_lineage.rs | 13 ++--------- .../src/lineage/main/build_fast_lineage.rs | 7 +++--- .../src/parse_guard/constants.rs | 2 +- .../sqlbuild-analysis/src/parse_guard/mod.rs | 2 +- .../src/semantic_validation/main.rs | 5 +--- scripts/check_parse_guard.py | 2 +- scripts/parse_guard/_helpers/scan.py | 2 +- scripts/parse_guard/constants.py | 4 +++- scripts/parse_guard/main/check.py | 4 ++-- .../scripts/parse_guard/_helpers/test_scan.py | 23 +++++++++++++++++++ 14 files changed, 42 insertions(+), 44 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index e29815c6ff..4e4ad91ae5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -76,12 +76,9 @@ fn analysed(input: &EnrichmentInput<'_>) -> Result { if let Some(schema) = analysis_schema(input) { options.insert("schema".to_owned(), schema); } - options.insert( - "complexityGuard".to_owned(), - serde_json::to_value(polyglot_complexity_guard()).map_err(|error| error.to_string())?, - ); - let options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options)) + let mut options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options)) .map_err(|error| format!("query analysis options: {error}"))?; + options.complexity_guard = Some(polyglot_complexity_guard()); let analysis = match analyze_query(&cleaned, options) { Ok(analysis) => analysis, Err(_) => return legacy(input, &cleaned), diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs index 330f9618fa..3296318703 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs @@ -2,8 +2,8 @@ use polyglot_sql::Dialect; -use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options; use crate::compiler::constants::FUNCTION_TYPE_DIALECTS; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; /// The error for a type the adapter's dialect cannot parse; adapters without a dialect pass. pub(crate) fn function_type_error( @@ -17,9 +17,8 @@ pub(crate) fn function_type_error( else { return Ok(None); }; - let options = guarded_parse_options().map_err(|error| error.to_string())?; Ok(Dialect::get(*dialect) - .parse_data_type_with_options(type_sql, &options) + .parse_data_type_with_options(type_sql, &polyglot_parse_options()) .err() .map(|error| { format!( diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs index bfe54fc9de..e29031ff03 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs @@ -1,2 +1 @@ pub(crate) mod function_types; -pub(crate) mod parse_options; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs deleted file mode 100644 index 3a1b66f829..0000000000 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! The Polyglot parse options SQLBuild's wheel proxy applies to every parse. - -use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; -use polyglot_sql::ParseOptions; - -/// The proxy's complexity guard, decoded as the wheel decodes it. -pub(crate) fn guarded_parse_options() -> Result { - Ok(polyglot_parse_options()) -} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs index bce222927c..eacc193e3f 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs @@ -15,12 +15,6 @@ use crate::lineage::models::{ FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSource, LineageTransformKind, }; -use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; - -/// The parse options SQLBuild's Polyglot proxy gives `parse_one`, decoded as the wheel does. -pub(crate) fn proxy_parse_options() -> Result { - Ok(polyglot_parse_options()) -} pub(crate) struct ParsedModel<'a> { pub(crate) query_sql: &'a str, @@ -28,7 +22,7 @@ pub(crate) struct ParsedModel<'a> { pub(crate) schema: &'a HashMap>, /// The parser dialect, or the dialect name Polyglot does not know. pub(crate) dialect: Result, - pub(crate) options: &'a Result, + pub(crate) options: &'a ParseOptions, } /// The model's lineage outcome; fails on a tree the lineage walk cannot read. @@ -37,10 +31,7 @@ pub(crate) fn parsed_model_lineage(model: &ParsedModel<'_>) -> Result dialect, Err(name) => return Ok(FastLineageOutcome::UnknownDialect(name.to_owned())), }; - let options: &ParseOptions = model - .options - .as_ref() - .map_err(|_| "the parse options do not build".to_owned())?; + let options: &ParseOptions = model.options; let physical = physical_resources(model.query_sql); let sql = normalized_sql(model.query_sql); let mut statements = match Dialect::get(dialect).parse_with_options(&sql, options) { diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs index d20bf68d63..5e1145f5be 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/build_fast_lineage.rs @@ -11,13 +11,12 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::analysis_route::main::analysis_route::analysis_route; use crate::analysis_route::main::fast_lineage::fast_lineage; use crate::analysis_route::models::AnalysisRoute; -use crate::lineage::_helpers::parsed_lineage::{ - ParsedModel, parsed_model_lineage, proxy_parse_options, -}; +use crate::lineage::_helpers::parsed_lineage::{ParsedModel, parsed_model_lineage}; use crate::lineage::_helpers::references::physical_resources; use crate::lineage::_helpers::stars::{schema_mapping, star_lineage}; use crate::lineage::main::parser_dialect::parser_dialect; use crate::lineage::models::{FastLineageModel, FastLineageOutcome, FastLineageRequest}; +use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options; use crate::semantic_validation::main::new_analysis_pool::new_analysis_pool; use crate::semantic_validation::models::ProjectCatalog; @@ -56,7 +55,7 @@ fn model_outcomes(request: &FastLineageRequest) -> Vec = proxy_parse_options(); + let options: ParseOptions = polyglot_parse_options(); request .models .par_iter() diff --git a/crates/sqlbuild-analysis/src/parse_guard/constants.rs b/crates/sqlbuild-analysis/src/parse_guard/constants.rs index db45e90082..1d65848990 100644 --- a/crates/sqlbuild-analysis/src/parse_guard/constants.rs +++ b/crates/sqlbuild-analysis/src/parse_guard/constants.rs @@ -1,4 +1,4 @@ //! Constants of parse_guard. -/// The deepest function-call nesting any SQLBuild parse accepts, as SQLQuack's guarded parse. +/// The deepest function-call nesting a Polyglot or SQLQuack shared or Snowflake parse accepts. pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; diff --git a/crates/sqlbuild-analysis/src/parse_guard/mod.rs b/crates/sqlbuild-analysis/src/parse_guard/mod.rs index add9304504..c636c77864 100644 --- a/crates/sqlbuild-analysis/src/parse_guard/mod.rs +++ b/crates/sqlbuild-analysis/src/parse_guard/mod.rs @@ -1,4 +1,4 @@ -//! The one function-call depth every SQL parse in SQLBuild allows, for Polyglot and SQLQuack alike. +//! One call depth for Polyglot and SQLQuack's shared and Snowflake parsers, not per-dialect ones. pub mod constants; pub mod main; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main.rs b/crates/sqlbuild-analysis/src/semantic_validation/main.rs index bd9e7446b5..5e0c5393df 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main.rs @@ -92,10 +92,7 @@ pub(super) fn validation_result(request: ValidationRequest) -> Result int: - """Check that native SQL parses use the shared parse guard.""" + """Check that Polyglot and SQLQuack's shared and Snowflake parses use the shared guard.""" return run_parse_guard_check(argv) diff --git a/scripts/parse_guard/_helpers/scan.py b/scripts/parse_guard/_helpers/scan.py index 4687f2d2f2..4693d761cf 100644 --- a/scripts/parse_guard/_helpers/scan.py +++ b/scripts/parse_guard/_helpers/scan.py @@ -1,4 +1,4 @@ -"""Find native SQL parses that fall back to a library's default function-call depth.""" +"""Find Polyglot and SQLQuack shared or Snowflake parses that use a library default depth.""" from __future__ import annotations diff --git a/scripts/parse_guard/constants.py b/scripts/parse_guard/constants.py index 0581a35178..d2408518d6 100644 --- a/scripts/parse_guard/constants.py +++ b/scripts/parse_guard/constants.py @@ -1,4 +1,4 @@ -"""Text that marks a native SQL parse using a library's default complexity guard.""" +"""Text that marks a guarded parse using a library's default depth.""" CRATES_DIRECTORY: str = "crates" SOURCE_SUFFIX: str = ".rs" @@ -11,12 +11,14 @@ r"max_function_call_depth: Some\((?!MAX_FUNCTION_CALL_DEPTH\))", r"const MAX_FUNCTION_CALL_DEPTH", r"polyglot_sql::(?:parse|transpile|validate)\(", + r"\bparse_data_type\(", r"Dialect::get(?:_by_name)?\([^;]*?\)\s*(?:\.[a-z_]+\([^;]*?\)\??\s*)*?\.parse\(", ) GUARDED_OPTION_LITERALS: tuple[str, ...] = ( "SchemaValidationOptions {", "AnalyzeQueryOptions {", "ValidationOptions {", + "ParseOptions {", ) GUARD_FIELD: str = "complexity_guard" TYPE_POSITION_SUFFIXES: tuple[str, ...] = ("->", "struct") diff --git a/scripts/parse_guard/main/check.py b/scripts/parse_guard/main/check.py index 9566a37aaf..b66ae63f97 100644 --- a/scripts/parse_guard/main/check.py +++ b/scripts/parse_guard/main/check.py @@ -7,13 +7,13 @@ def run_parse_guard_check(argv: list[str] | None = None) -> int: - """Fail when a native SQL parse uses a library's default function-call depth.""" + """Fail when a Polyglot or SQLQuack shared or Snowflake parse uses a library default depth.""" return run_root_check( argv=argv, description=__doc__, errors_under=get_parse_guard_errors, messages=( "Parse with sqlbuild_analysis::parse_guard's options, not a library default:", - "Every native SQL parse uses the shared parse guard.", + "Polyglot and SQLQuack's shared and Snowflake parsers all use the shared guard.", ), ) diff --git a/tests/unit/scripts/parse_guard/_helpers/test_scan.py b/tests/unit/scripts/parse_guard/_helpers/test_scan.py index eed3e400e4..82952a40f4 100644 --- a/tests/unit/scripts/parse_guard/_helpers/test_scan.py +++ b/tests/unit/scripts/parse_guard/_helpers/test_scan.py @@ -39,12 +39,35 @@ text="fn d() { let o = AnalyzeQueryOptions { dialect, schema: Some(S { t }) }; }", expected_sites=("x.rs:1 AnalyzeQueryOptions {",), ), + UnguardedSiteTestCase( + description="Polyglot parse options that fill the rest from the default", + text="fn h() { let o = ParseOptions { ..Default::default() }; }", + expected_sites=("x.rs:1 ParseOptions {",), + ), + UnguardedSiteTestCase( + description="a data type parse without options", + text="fn i() { Dialect::get(d).parse_data_type(spelling) }", + expected_sites=(r"x.rs:1 \bparse_data_type\(",), + ), + UnguardedSiteTestCase( + description=( + "SQLQuack's per-dialect parsers are excluded: they take no options and keep their " + "own depth limits until SQLQuack accepts options for them" + ), + text=( + "fn k() { sqlquack_syntax_postgres::parse_strict(sql) }\n" + "fn l() { sqlquack_syntax_bigquery::parse_type(spelling) }\n" + "fn m(parser: &dyn NativeDialect) { parser.parse(sql) }\n" + ), + expected_sites=(), + ), UnguardedSiteTestCase( description="the shared options and constant", text=( "fn e() -> AnalyzeQueryOptions { AnalyzeQueryOptions { complexity_guard: g } }\n" "fn f() { Dialect::get(d).parse_with_options(sql, &polyglot_parse_options()); }\n" "fn g() { R { max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH) } }\n" + "fn j() { d.parse_data_type_with_options(t, &ParseOptions { complexity_guard: g }) }\n" ), expected_sites=(), ), From f79eac92e3fd8b48964872c262fc0a1baa4d0978 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:38:41 +0000 Subject: [PATCH 5/5] test(rules): expect built-in model rules to parse calls as deep as the compiler The built-in model rules' query facts now share the compiler's 256 function-call depth, so SQL nested 130 calls deep, once beyond their 128 budget, is evaluated. The unevaluated-resource case for SQBRMODEL101 becomes a case that the rule evaluates it; SQLClean rules keep their own budget and their unevaluated cases stay. --- .../cli/commands/main/rules/_test_types.py | 9 ++++ .../commands/main/rules/test_unevaluated.py | 43 +++++++++++++++---- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 87177244e5..f230c7979c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -47,6 +47,15 @@ class SnowflakeBodyShapeCase: expected_format_exit: int = 0 +@dataclass(frozen=True) +class SharedDepthModelRuleCase: + description: str + template: str + rule: str + expected_evaluated_models: int + expected_unevaluated_resources: int + + @dataclass(frozen=True) class UnevaluatedResourceCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py index 4939d1b9fc..0ac0eb5ef6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -11,6 +11,7 @@ EvaluatedModelRulesCase, EvaluatedPredicateCase, GroupedRulesCase, + SharedDepthModelRuleCase, SnowflakeBodyShapeCase, UnevaluatedResourceCase, UnevaluatedRuleCase, @@ -48,14 +49,6 @@ ), ), ), - UnevaluatedResourceCase( - "built-in model rule beyond the parser budget", - "models/staging/orders.sql", - "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", - expected_evaluated_models=0, - expected_reason="could not parse models/staging/orders.sql for rules", - rule="SQBRMODEL101", - ), UnevaluatedResourceCase( "SQL-test guard", "tests/unit/test_orders.sql", @@ -109,6 +102,40 @@ def test_given_guarded_resource_when_running_rules_then_reports_failure_and_cove ) +@pytest.mark.parametrize( + "test_case", + [ + SharedDepthModelRuleCase( + "a built-in model rule parses calls nested as deep as the compiler accepts", + "MODEL (description 'Test model orders.');\nSELECT {expression} AS order_id", + "SQBRMODEL101", + expected_evaluated_models=1, + expected_unevaluated_resources=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_calls_nested_within_the_shared_depth_when_running_model_rules_then_evaluates( + test_case: SharedDepthModelRuleCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + resource_path="models/staging/orders.sql", + resource_template=test_case.template, + selected_rules=(test_case.rule,), + ) + + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", test_case.rule + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["unevaluated_resources"] == test_case.expected_unevaluated_resources + assert payload["evaluated_models"] == test_case.expected_evaluated_models + + @pytest.mark.parametrize( "test_case", [