diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index 9bb85d05c..0313f842c 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -344,6 +344,20 @@ jobs: "${python[@]}" -c 'import sqlquack result = sqlquack.convert(sql="SELECT 1 AS total", options=sqlquack.ConvertOptions(no_config=True)) assert result.sql == "SELECT CAST(1 AS DECIMAL(1, 0)) AS \"TOTAL\"", result.sql' + # Deep function-call nesting on a small thread stack, where Windows threads are smallest: + # 511 levels read at the 512 ceiling; 257 at the default is a clean error, not a crash. + "${python[@]}" -c 'import sqlquack, threading + def nested(levels): + return "SELECT " + "ABS(" * levels + "1" + ")" * levels + found = [] + threading.stack_size(256 * 1024) + def check(): + found.append(sqlquack.check(sql=nested(511), max_call_depth=512).ok) + found.append(sqlquack.check(sql=nested(257)).diagnostics[0].code) + thread = threading.Thread(target=check) + thread.start() + thread.join() + assert found == [True, "SQ0107"], found' # A transient network error in the upload would otherwise cost the whole build, so it retries # once, replacing any artifact the failed attempt left behind. - id: upload-wheel diff --git a/Cargo.lock b/Cargo.lock index 3bcb79250..c8957cf25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1551,6 +1551,7 @@ dependencies = [ "serde_json", "sqlquack", "sqlquack-diagnostics", + "sqlquack-parse", "sqlquack-policy", "sqlquack-semantics", ] diff --git a/crates/sqlquack-convert/src/snowflake/analysis/main/analyze.rs b/crates/sqlquack-convert/src/snowflake/analysis/main/analyze.rs index 2d14b2b90..2b68d086f 100644 --- a/crates/sqlquack-convert/src/snowflake/analysis/main/analyze.rs +++ b/crates/sqlquack-convert/src/snowflake/analysis/main/analyze.rs @@ -1,7 +1,7 @@ //! `analyze`: parse, classify, convert or refuse. use sqlquack_diagnostics::classification::main::classify::classify; -use sqlquack_parse::parsing::main::parse_source::parse_source; +use sqlquack_parse::parsing::main::parse_source_within::parse_source_within; use sqlquack_syntax::Expression; use std::time::Instant; @@ -36,7 +36,7 @@ pub fn analyze( quoted_exclusions: quoted_exclusions(sql), ..options.clone() }; - let statements = match parse_source(sql, Dialect::Snowflake) { + let statements = match parse_source_within(sql, Dialect::Snowflake, options.max_call_depth) { Ok(statements) => statements, Err(failure) => { let refusal = refuse_unparsed(&failure, &lines); diff --git a/crates/sqlquack-convert/src/snowflake/analysis/main/analyze_function.rs b/crates/sqlquack-convert/src/snowflake/analysis/main/analyze_function.rs index 57028b1c9..11c65a311 100644 --- a/crates/sqlquack-convert/src/snowflake/analysis/main/analyze_function.rs +++ b/crates/sqlquack-convert/src/snowflake/analysis/main/analyze_function.rs @@ -9,7 +9,7 @@ use sqlquack_diagnostics::refusals::main::refuse_unparsed::refuse_unparsed; use sqlquack_diagnostics::refusals::models::{Refusal, RefusalCategory}; use sqlquack_diagnostics::refusals::types::ConstructPolicy; use sqlquack_diagnostics::snowflake::classification::main::cast_type_name::cast_type_name; -use sqlquack_parse::parsing::main::parse_source::parse_source; +use sqlquack_parse::parsing::main::parse_source_within::parse_source_within; use sqlquack_semantics::snowflake::typing::models::SnowflakeType; use sqlquack_semantics::snowflake::typing::types::{FunctionReturns, FunctionSpec, Schema}; use sqlquack_syntax::Dialect; @@ -42,7 +42,7 @@ pub fn analyze_function( ..options.clone() }; let lines = SourceLines::shifted(body, prefix.chars().count()); - let statements = parse_source(&sql, Dialect::Snowflake) + let statements = parse_source_within(&sql, Dialect::Snowflake, options.max_call_depth) .map_err(|failure| vec![refuse_unparsed(&failure, &lines)])?; let [statement] = statements.as_slice() else { return Err(vec![Refusal { diff --git a/crates/sqlquack-convert/src/snowflake/analysis/models.rs b/crates/sqlquack-convert/src/snowflake/analysis/models.rs index 85c280f1a..af55655f8 100644 --- a/crates/sqlquack-convert/src/snowflake/analysis/models.rs +++ b/crates/sqlquack-convert/src/snowflake/analysis/models.rs @@ -33,4 +33,6 @@ pub struct ConversionChoices { pub strict: bool, /// A Python runtime registers the project's Python functions. pub python_runtime: bool, + /// Function calls nested deeper than this are a parse error; the parser's default when absent. + pub max_call_depth: Option, } diff --git a/crates/sqlquack-convert/src/snowflake/conversion/models.rs b/crates/sqlquack-convert/src/snowflake/conversion/models.rs index c24201ac5..85741552b 100644 --- a/crates/sqlquack-convert/src/snowflake/conversion/models.rs +++ b/crates/sqlquack-convert/src/snowflake/conversion/models.rs @@ -46,6 +46,8 @@ pub struct ConversionOptions { pub overrides: BTreeMap, /// Override names to the SHA-256 of the body recorded cases verified, which strict mode allows. pub verified_overrides: BTreeMap, + /// Function calls nested deeper than this are a parse error; the parser's default when absent. + pub max_call_depth: Option, } /// A user's DuckDB SQL standing in for a Snowflake function, written in SQLBuild's FUNCTION() format. diff --git a/crates/sqlquack-convert/src/snowflake/session/main/configured_choices.rs b/crates/sqlquack-convert/src/snowflake/session/main/configured_choices.rs index 58f2d425f..1beb4bc9c 100644 --- a/crates/sqlquack-convert/src/snowflake/session/main/configured_choices.rs +++ b/crates/sqlquack-convert/src/snowflake/session/main/configured_choices.rs @@ -28,6 +28,7 @@ pub fn configured_choices( allow: config.allow, strict, python_runtime: config.project.python_runtime, + max_call_depth: None, }; Ok((choices, session)) } diff --git a/crates/sqlquack-convert/src/snowflake/session/main/configured_options.rs b/crates/sqlquack-convert/src/snowflake/session/main/configured_options.rs index 385dcf6e1..de4347d0b 100644 --- a/crates/sqlquack-convert/src/snowflake/session/main/configured_options.rs +++ b/crates/sqlquack-convert/src/snowflake/session/main/configured_options.rs @@ -11,6 +11,7 @@ use sqlquack_policy::policy::main::evidence_policy::evidence_policy; pub fn configured_options(choices: &ConversionChoices) -> ConversionOptions { let evidence = evidence_policy(); let mut options = builtin_conversion_options(choices.strict); + options.max_call_depth = choices.max_call_depth; if let Some(budget) = evidence.conversion_budget { options.budget = ConversionBudget { milliseconds: budget.milliseconds, diff --git a/crates/sqlquack-diagnostics/src/diagnostics/constants.rs b/crates/sqlquack-diagnostics/src/diagnostics/constants.rs index 98ec5709a..4a753fc48 100644 --- a/crates/sqlquack-diagnostics/src/diagnostics/constants.rs +++ b/crates/sqlquack-diagnostics/src/diagnostics/constants.rs @@ -261,12 +261,13 @@ pub const CODES: &[CodeEntry] = &[ alias: Some("E_GUARD_NESTING_DEPTH_EXCEEDED"), title: "brackets nest deeper than the parser accepts", help: Some( - "remove redundant parentheses, or move nested parts into common table expressions", + "remove redundant parentheses, or move nested parts into common table expressions; \ + the limit of 512 is fixed, and --max-call-depth can't raise it", ), explanation: "Parentheses, brackets and braces nest deeper at the span than the \ - parser's limit. The limit keeps parsing from exhausting the stack. Redundant \ - parentheses can be removed; deeply nested subqueries can usually become common \ - table expressions.", + parser's limit of 512. The limit keeps parsing from exhausting the stack, and no \ + setting raises it. Redundant parentheses can be removed; deeply nested subqueries \ + can usually become common table expressions.", example: r#"python3 -c "print('SELECT ' + '(' * 600 + '1' + ')' * 600)" | sqlquack check"#, }, CodeEntry { @@ -285,10 +286,13 @@ pub const CODES: &[CodeEntry] = &[ name: "ast_depth_exceeded", alias: Some("E_GUARD_AST_DEPTH_EXCEEDED"), title: "the parsed statement nests deeper than SQLQuack accepts", - help: Some("move nested parts into common table expressions"), - explanation: "The statement parsed, but its tree nests deeper than the limit SQLQuack \ - places on later analysis, which keeps conversion from exhausting the stack. \ - Restructure the deepest part of the statement.", + help: Some( + "move nested parts into common table expressions; the limit of 512 is fixed, and \ + --max-call-depth can't raise it", + ), + explanation: "The statement parsed, but its tree nests deeper than the limit of 512 \ + SQLQuack places on later analysis, which keeps conversion from exhausting the stack. \ + No setting raises it. Restructure the deepest part of the statement.", example: r#"python3 -c "print('SELECT ' + 'CASE WHEN TRUE THEN ' * 600 + '1' + ' END' * 600)" | sqlquack check"#, }, CodeEntry { @@ -306,10 +310,15 @@ pub const CODES: &[CodeEntry] = &[ name: "function_nesting_depth_exceeded", alias: Some("E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED"), title: "function calls nest deeper than SQLQuack accepts", - help: Some("compute inner calls in a common table expression first"), + help: Some( + "compute inner calls in a common table expression first, or raise the limit, up to \ + 512, with --max-call-depth (command line) or max_call_depth= (Python)", + ), explanation: "Function calls nest inside each other deeper than SQLQuack's limit at \ - the span. Compute the innermost calls in a common table expression or a subquery \ - and refer to their results.", + the span, 256 by default. Compute the innermost calls in a common table expression \ + or a subquery and refer to their results, or raise the limit with \ + `--max-call-depth` on the command line or `max_call_depth=` in Python. The limit \ + goes up to 512, the parser's fixed nesting limit.", example: r#"python3 -c "print('SELECT ' + 'ABS(' * 300 + '1' + ')' * 300)" | sqlquack check"#, }, CodeEntry { diff --git a/crates/sqlquack-lint/src/lint/_helpers/engine.rs b/crates/sqlquack-lint/src/lint/_helpers/engine.rs index c0dcb2a32..86348176a 100644 --- a/crates/sqlquack-lint/src/lint/_helpers/engine.rs +++ b/crates/sqlquack-lint/src/lint/_helpers/engine.rs @@ -387,10 +387,13 @@ pub(crate) fn lint(request: LintRequest) -> Result { } = parse( &request.sql, dialect, - request - .fixture_harness - .as_ref() - .and_then(|harness| harness.empty_relation.as_deref()), + ( + request + .fixture_harness + .as_ref() + .and_then(|harness| harness.empty_relation.as_deref()), + request.max_function_call_depth, + ), )?; let external_identifiers: HashSet = request .external_identifiers diff --git a/crates/sqlquack-lint/src/lint/_helpers/parsing.rs b/crates/sqlquack-lint/src/lint/_helpers/parsing.rs index df1b38470..0859173ec 100644 --- a/crates/sqlquack-lint/src/lint/_helpers/parsing.rs +++ b/crates/sqlquack-lint/src/lint/_helpers/parsing.rs @@ -2,6 +2,9 @@ use std::collections::HashSet; +use sqlquack_parse::parsing::constants::MAX_FUNCTION_CALL_DEPTH; +use sqlquack_parse::parsing::main::call_depth_hint::call_depth_hint; +use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use sqlquack_syntax::tokens::models::{Token, TokenType}; use sqlquack_syntax::{ComplexityGuardOptions, Expression, ExpressionWalk, ParseOptions}; @@ -9,19 +12,20 @@ use crate::dialects::main::profile::profile; use crate::dialects::models::LintProfile; use crate::lint::models::{LintDialect, ParsedSql}; -const MAX_LINT_FUNCTION_CALL_DEPTH: usize = 128; - /// Tokenize and parse `sql`, reading a harness's `FROM ()` as a bare relation. pub(crate) fn parse( sql: &str, dialect: LintDialect, - empty_relation: Option<&str>, + (empty_relation, max_call_depth): (Option<&str>, Option), ) -> Result { + let max_call_depth: usize = + checked_call_depth(max_call_depth.unwrap_or(MAX_FUNCTION_CALL_DEPTH)) + .map_err(|reason| format!("max_function_call_depth: {reason}"))?; let profile: LintProfile = profile(dialect); let tokens: Vec = (profile.tokens)(sql)?; let options = ParseOptions { complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: Some(MAX_LINT_FUNCTION_CALL_DEPTH), + max_function_call_depth: Some(max_call_depth), ..Default::default() }), }; @@ -33,7 +37,10 @@ pub(crate) fn parse( } else { &parse_text }; - let statements: Vec = (profile.statements)(parse_text, &options)?; + let statements: Vec = + (profile.statements)(parse_text, &options).map_err(|message| { + call_depth_hint(message, max_call_depth, Some("max_function_call_depth")) + })?; for statement in &statements { if statement .dfs() diff --git a/crates/sqlquack-lint/src/lint/main/parse_sql.rs b/crates/sqlquack-lint/src/lint/main/parse_sql.rs index 5d9d0110b..2eed4aec2 100644 --- a/crates/sqlquack-lint/src/lint/main/parse_sql.rs +++ b/crates/sqlquack-lint/src/lint/main/parse_sql.rs @@ -7,5 +7,5 @@ pub fn parse_sql(sql: &str, dialect: &str) -> Result { let Some(parsed) = crate::dialects::main::lint_dialect::lint_dialect(dialect)? else { return Err(format!("SQLQuack does not parse the {dialect} dialect yet")); }; - crate::lint::_helpers::parsing::parse(sql, parsed, None) + crate::lint::_helpers::parsing::parse(sql, parsed, (None, None)) } diff --git a/crates/sqlquack-lint/src/lint/models.rs b/crates/sqlquack-lint/src/lint/models.rs index 71f2d03ba..58e0f8a7e 100644 --- a/crates/sqlquack-lint/src/lint/models.rs +++ b/crates/sqlquack-lint/src/lint/models.rs @@ -97,6 +97,9 @@ pub struct LintRequest { pub max_literal_length: Option, #[serde(default)] pub max_ranking_order_by: Option, + /// Function calls nested deeper than this are a parse error; 256 when absent, at most 512. + #[serde(default)] + pub max_function_call_depth: Option, #[serde(default)] pub relation_keys: HashMap>>, #[serde(default)] diff --git a/crates/sqlquack-lint/src/lint/tests.rs b/crates/sqlquack-lint/src/lint/tests.rs index 378aaa1ca..5f25fec67 100644 --- a/crates/sqlquack-lint/src/lint/tests.rs +++ b/crates/sqlquack-lint/src/lint/tests.rs @@ -4,6 +4,8 @@ mod batch_lint; mod cross_joins; #[path = "tests/test_empty_fixture_lint.rs"] mod empty_fixture_lint; +#[path = "tests/test_function_depth.rs"] +mod function_depth; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_join_predicates.rs"] diff --git a/crates/sqlquack-lint/src/lint/tests/helpers.rs b/crates/sqlquack-lint/src/lint/tests/helpers.rs index 2174fcb88..2eefe7d28 100644 --- a/crates/sqlquack-lint/src/lint/tests/helpers.rs +++ b/crates/sqlquack-lint/src/lint/tests/helpers.rs @@ -19,14 +19,21 @@ pub(crate) fn nested_from_sql(depth: usize) -> String { } pub(crate) fn diagnostics(sql: &str) -> Result, String> { - let response = lint_json( - &json!({ - "version": 1, - "sql": sql, - "dialect": "snowflake" - }) - .to_string(), - )?; + diagnostics_within(sql, None) +} + +/// The Snowflake diagnostics of `sql`, with the request's `max_function_call_depth`, null when absent. +pub(crate) fn diagnostics_within( + sql: &str, + max_function_call_depth: Option, +) -> Result, String> { + let request = json!({ + "version": 1, + "sql": sql, + "dialect": "snowflake", + "max_function_call_depth": max_function_call_depth + }); + let response = lint_json(&request.to_string())?; diagnostic_values(&response) } diff --git a/crates/sqlquack-lint/src/lint/tests/test_function_depth.rs b/crates/sqlquack-lint/src/lint/tests/test_function_depth.rs new file mode 100644 index 000000000..66741b4ad --- /dev/null +++ b/crates/sqlquack-lint/src/lint/tests/test_function_depth.rs @@ -0,0 +1,82 @@ +use crate::lint::tests::{helpers, test_types}; + +#[test] +fn given_production_function_depth_when_linting_then_bounded_parser_accepts_it() { + let test_cases = [ + test_types::FunctionDepthTestCase { + description: "production depth remains supported", + depth: 65, + max_function_call_depth: None, + expected_diagnostic_count: 0, + }, + test_types::FunctionDepthTestCase { + description: "the default limit reads 256 nested calls", + depth: 256, + max_function_call_depth: None, + expected_diagnostic_count: 0, + }, + test_types::FunctionDepthTestCase { + description: "a raised limit of 300 reads 300 nested calls", + depth: 300, + max_function_call_depth: Some(300), + expected_diagnostic_count: 0, + }, + ]; + for test_case in test_cases { + let sql = helpers::nested_function_sql(test_case.depth); + let diagnostics = helpers::diagnostics_within(&sql, test_case.max_function_call_depth) + .unwrap_or_else(|error| { + panic!("{}: {error}", test_case.description); + }); + assert_eq!( + diagnostics.len(), + test_case.expected_diagnostic_count, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_excessive_function_depth_when_linting_then_complexity_guard_rejects_it() { + let test_cases = [ + test_types::FunctionDepthFailureTestCase { + description: "the default limit refuses 257 nested calls and says how to raise it", + depth: 257, + max_function_call_depth: None, + expected_error_end: "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value 257 exceeds \ + configured limit 256; raise max_function_call_depth, up to 512", + }, + test_types::FunctionDepthFailureTestCase { + description: "a lowered limit of 10 refuses 11 nested calls", + depth: 11, + max_function_call_depth: Some(10), + expected_error_end: "value 11 exceeds configured limit 10; raise \ + max_function_call_depth, up to 512", + }, + test_types::FunctionDepthFailureTestCase { + description: "a limit past the ceiling is refused, naming the ceiling", + depth: 300, + max_function_call_depth: Some(1000), + expected_error_end: "max_function_call_depth: the function-call depth limit must be \ + between 1 and 512, SQLQuack's fixed nesting ceiling, not 1000", + }, + test_types::FunctionDepthFailureTestCase { + description: "at the ceiling of 512, 600 nested calls are refused, naming it as fixed", + depth: 600, + max_function_call_depth: Some(512), + expected_error_end: "E_GUARD_NESTING_DEPTH_EXCEEDED: value 513 exceeds configured \ + limit 512; 512 is SQLQuack's fixed nesting ceiling, which no setting raises", + }, + ]; + for test_case in test_cases { + let sql = helpers::nested_function_sql(test_case.depth); + let error = helpers::diagnostics_within(&sql, test_case.max_function_call_depth) + .expect_err(test_case.description); + assert!( + error.ends_with(test_case.expected_error_end), + "{}: {error}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs b/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs index 2f0720cdd..f5b29b248 100644 --- a/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs +++ b/crates/sqlquack-lint/src/lint/tests/test_sql_lint.rs @@ -4,52 +4,6 @@ use crate::lint::main::engine::lint_json; use crate::lint::tests::helpers::fixture_harness; use crate::lint::tests::{helpers, test_types}; -#[test] -fn given_production_function_depth_when_linting_then_bounded_parser_accepts_it() { - let test_cases = [test_types::FunctionDepthTestCase { - description: "production depth remains supported", - depth: 65, - expected_diagnostic_count: 0, - }]; - for test_case in test_cases { - let sql = helpers::nested_function_sql(test_case.depth); - let diagnostics = helpers::diagnostics(&sql).unwrap_or_else(|error| { - panic!("{}: {error}", test_case.description); - }); - assert_eq!( - diagnostics.len(), - test_case.expected_diagnostic_count, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_excessive_function_depth_when_linting_then_complexity_guard_rejects_it() { - let test_cases = [test_types::FunctionDepthFailureTestCase { - description: "excessive depth remains bounded", - depth: 129, - expected_code: "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED", - expected_limit: "configured limit 128", - }]; - for test_case in test_cases { - let sql = helpers::nested_function_sql(test_case.depth); - let error = - helpers::diagnostics(&sql).expect_err("excessive nested calls should remain bounded"); - assert!( - error.contains(test_case.expected_code), - "{}", - test_case.description - ); - assert!( - error.contains(test_case.expected_limit), - "{}", - test_case.description - ); - } -} - #[test] fn given_deeply_nested_from_parentheses_when_linting_then_each_parser_refuses_cleanly() { let test_cases = [ diff --git a/crates/sqlquack-lint/src/lint/tests/test_types.rs b/crates/sqlquack-lint/src/lint/tests/test_types.rs index 0c9573818..fd32f084c 100644 --- a/crates/sqlquack-lint/src/lint/tests/test_types.rs +++ b/crates/sqlquack-lint/src/lint/tests/test_types.rs @@ -98,14 +98,18 @@ pub(crate) struct InvalidRuleTestCase { pub(crate) struct FunctionDepthTestCase { pub description: &'static str, pub depth: usize, + /// The request's `max_function_call_depth`; absent for the default. + pub max_function_call_depth: Option, pub expected_diagnostic_count: usize, } pub(crate) struct FunctionDepthFailureTestCase { pub description: &'static str, pub depth: usize, - pub expected_code: &'static str, - pub expected_limit: &'static str, + /// The request's `max_function_call_depth`; absent for the default. + pub max_function_call_depth: Option, + /// How the error ends: the guard's value and limit, then how to raise it or why it can't be. + pub expected_error_end: &'static str, } pub(crate) struct NestingRefusalTestCase { diff --git a/crates/sqlquack-parse/src/parsing/_helpers/guarded.rs b/crates/sqlquack-parse/src/parsing/_helpers/guarded.rs index 555f453d0..6180e1586 100644 --- a/crates/sqlquack-parse/src/parsing/_helpers/guarded.rs +++ b/crates/sqlquack-parse/src/parsing/_helpers/guarded.rs @@ -2,14 +2,17 @@ use sqlquack_syntax::{CallMode, ComplexityGuardOptions, Dialect, Error, Expression, ParseOptions}; -use crate::parsing::constants::MAX_FUNCTION_CALL_DEPTH; use sqlquack_syntax::tokens::errors::ParseFailure; use sqlquack_syntax::tokens::models::SourceLocation; -pub(crate) fn parse_guarded(sql: &str, dialect: Dialect) -> Result, ParseFailure> { +pub(crate) fn parse_guarded( + sql: &str, + dialect: Dialect, + max_call_depth: usize, +) -> Result, ParseFailure> { let options = ParseOptions { complexity_guard: Some(ComplexityGuardOptions { - max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + max_function_call_depth: Some(max_call_depth), ..ComplexityGuardOptions::default() }), }; diff --git a/crates/sqlquack-parse/src/parsing/constants.rs b/crates/sqlquack-parse/src/parsing/constants.rs index 92e91b6b3..eb47f60ec 100644 --- a/crates/sqlquack-parse/src/parsing/constants.rs +++ b/crates/sqlquack-parse/src/parsing/constants.rs @@ -2,3 +2,12 @@ /// Nested function calls allowed per statement, above the parser's default, for generated SQL. pub const MAX_FUNCTION_CALL_DEPTH: usize = 256; + +/// The highest function-call limit a caller may set, the fixed nesting limit every parse keeps. +pub const MAX_CALL_DEPTH_CEILING: usize = 512; + +/// The guard code of a parse refused for function calls nested past the limit. +pub const FUNCTION_NESTING_CODE: &str = "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED"; + +/// The guard code of a parse refused for brackets nested past the fixed ceiling. +pub const NESTING_CODE: &str = "E_GUARD_NESTING_DEPTH_EXCEEDED"; diff --git a/crates/sqlquack-parse/src/parsing/main/call_depth_hint.rs b/crates/sqlquack-parse/src/parsing/main/call_depth_hint.rs new file mode 100644 index 000000000..94e932e54 --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/main/call_depth_hint.rs @@ -0,0 +1,20 @@ +//! `call_depth_hint`: what a refusal for nesting past a limit tells the caller. + +use crate::parsing::constants::{FUNCTION_NESTING_CODE, MAX_CALL_DEPTH_CEILING, NESTING_CODE}; + +/// `message` naming the fixed ceiling it met, or how to raise `limit` through `setting`. +pub fn call_depth_hint(message: String, limit: usize, setting: Option<&str>) -> String { + let function_nesting: bool = message.contains(FUNCTION_NESTING_CODE); + if message.contains(NESTING_CODE) || (function_nesting && limit >= MAX_CALL_DEPTH_CEILING) { + return format!( + "{message}; {MAX_CALL_DEPTH_CEILING} is SQLQuack's fixed nesting ceiling, which no \ + setting raises" + ); + } + match (function_nesting, setting) { + (true, Some(setting)) => { + format!("{message}; raise {setting}, up to {MAX_CALL_DEPTH_CEILING}") + } + _ => message, + } +} diff --git a/crates/sqlquack-parse/src/parsing/main/checked_call_depth.rs b/crates/sqlquack-parse/src/parsing/main/checked_call_depth.rs new file mode 100644 index 000000000..12cdf3636 --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/main/checked_call_depth.rs @@ -0,0 +1,14 @@ +//! `checked_call_depth`: a function-call limit a caller asked for, within what the parser allows. + +use crate::parsing::constants::MAX_CALL_DEPTH_CEILING; + +/// `depth` when it is between 1 and the ceiling, or why it is refused, naming the ceiling. +pub fn checked_call_depth(depth: usize) -> Result { + match depth { + 1..=MAX_CALL_DEPTH_CEILING => Ok(depth), + _ => Err(format!( + "the function-call depth limit must be between 1 and {MAX_CALL_DEPTH_CEILING}, \ + SQLQuack's fixed nesting ceiling, not {depth}" + )), + } +} diff --git a/crates/sqlquack-parse/src/parsing/main/mod.rs b/crates/sqlquack-parse/src/parsing/main/mod.rs index 2712aaffa..834563bad 100644 --- a/crates/sqlquack-parse/src/parsing/main/mod.rs +++ b/crates/sqlquack-parse/src/parsing/main/mod.rs @@ -1 +1,4 @@ +pub mod call_depth_hint; +pub mod checked_call_depth; pub mod parse_source; +pub mod parse_source_within; diff --git a/crates/sqlquack-parse/src/parsing/main/parse_source.rs b/crates/sqlquack-parse/src/parsing/main/parse_source.rs index 8d06db73d..dbb9a0941 100644 --- a/crates/sqlquack-parse/src/parsing/main/parse_source.rs +++ b/crates/sqlquack-parse/src/parsing/main/parse_source.rs @@ -4,8 +4,9 @@ use sqlquack_syntax::tokens::errors::ParseFailure; use sqlquack_syntax::{Dialect, Expression}; use crate::parsing::_helpers::guarded::parse_guarded; +use crate::parsing::constants::MAX_FUNCTION_CALL_DEPTH; /// Parses source SQL, which may hold several statements, in its own dialect. pub fn parse_source(sql: &str, dialect: Dialect) -> Result, ParseFailure> { - parse_guarded(sql, dialect) + parse_guarded(sql, dialect, MAX_FUNCTION_CALL_DEPTH) } diff --git a/crates/sqlquack-parse/src/parsing/main/parse_source_within.rs b/crates/sqlquack-parse/src/parsing/main/parse_source_within.rs new file mode 100644 index 000000000..1bbb1aad9 --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/main/parse_source_within.rs @@ -0,0 +1,27 @@ +//! `parse_source_within`: source SQL to statements, with the caller's function-call limit. + +use sqlquack_syntax::tokens::errors::ParseFailure; +use sqlquack_syntax::{Dialect, Expression}; + +use crate::parsing::_helpers::guarded::parse_guarded; +use crate::parsing::constants::MAX_FUNCTION_CALL_DEPTH; +use crate::parsing::main::call_depth_hint::call_depth_hint; +use crate::parsing::main::checked_call_depth::checked_call_depth; + +/// Parses source SQL as `parse_source` does, refusing calls nested deeper than `max_call_depth`. +pub fn parse_source_within( + sql: &str, + dialect: Dialect, + max_call_depth: Option, +) -> Result, ParseFailure> { + let limit = checked_call_depth(max_call_depth.unwrap_or(MAX_FUNCTION_CALL_DEPTH)).map_err( + |message| ParseFailure { + message, + location: None, + }, + )?; + parse_guarded(sql, dialect, limit).map_err(|failure| ParseFailure { + message: call_depth_hint(failure.message, limit, None), + ..failure + }) +} diff --git a/crates/sqlquack-parse/src/parsing/mod.rs b/crates/sqlquack-parse/src/parsing/mod.rs index b9f23d9c3..89776874d 100644 --- a/crates/sqlquack-parse/src/parsing/mod.rs +++ b/crates/sqlquack-parse/src/parsing/mod.rs @@ -3,3 +3,6 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlquack-parse/src/parsing/tests.rs b/crates/sqlquack-parse/src/parsing/tests.rs new file mode 100644 index 000000000..b8e0b879b --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_parse_source.rs"] +mod test_parse_source; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlquack-parse/src/parsing/tests/test_parse_source.rs b/crates/sqlquack-parse/src/parsing/tests/test_parse_source.rs new file mode 100644 index 000000000..40a5b2217 --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/tests/test_parse_source.rs @@ -0,0 +1,137 @@ +//! Function-call nesting: refused cleanly at the caller's limit, never by the thread's stack. + +use sqlquack_syntax::Dialect; + +use crate::parsing::main::checked_call_depth::checked_call_depth; +use crate::parsing::main::parse_source_within::parse_source_within; +use crate::parsing::tests::test_types::{CallDepthTestCase, CheckedDepthTestCase}; + +/// A thread stack smaller than any SQLQuack runs on, so passing shows the stack grows on demand. +const SMALL_STACK_BYTES: usize = 256 * 1024; + +#[test] +fn given_nested_function_calls_when_parsed_on_a_small_stack_then_the_limit_decides_cleanly() { + let test_cases = [ + CallDepthTestCase { + description: "the default limit reads 256 nested calls", + limit: None, + levels: 256, + expected_outcome: Ok(1), + }, + CallDepthTestCase { + description: "the default limit refuses 257 nested calls and names the limit", + limit: None, + levels: 257, + expected_outcome: Err( + "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value 257 exceeds configured limit 256", + ), + }, + CallDepthTestCase { + description: "a raised limit of 300 reads 300 nested calls", + limit: Some(300), + levels: 300, + expected_outcome: Ok(1), + }, + CallDepthTestCase { + description: "a raised limit of 300 refuses 301 nested calls", + limit: Some(300), + levels: 301, + expected_outcome: Err("value 301 exceeds configured limit 300"), + }, + CallDepthTestCase { + description: "the ceiling of 512 reads 511 nested calls inside a SELECT", + limit: Some(512), + levels: 511, + expected_outcome: Ok(1), + }, + CallDepthTestCase { + description: "a limit of 1000 is refused, naming the ceiling of 512", + limit: Some(1000), + levels: 1000, + expected_outcome: Err( + "must be between 1 and 512, SQLQuack's fixed nesting ceiling, not 1000", + ), + }, + CallDepthTestCase { + description: "at the ceiling, 600 nested calls are refused, naming the ceiling as fixed", + limit: Some(512), + levels: 600, + expected_outcome: Err( + "E_GUARD_NESTING_DEPTH_EXCEEDED: value 513 exceeds configured limit 512; 512 is \ + SQLQuack's fixed nesting ceiling, which no setting raises", + ), + }, + CallDepthTestCase { + description: "a lowered limit of 10 refuses 11 nested calls", + limit: Some(10), + levels: 11, + expected_outcome: Err("value 11 exceeds configured limit 10"), + }, + ]; + + for test_case in test_cases { + let sql = format!( + "SELECT {}1{}", + "ABS(".repeat(test_case.levels), + ")".repeat(test_case.levels) + ); + let limit = test_case.limit; + let outcome: Result = std::thread::Builder::new() + .stack_size(SMALL_STACK_BYTES) + .spawn(move || { + parse_source_within(&sql, Dialect::Snowflake, limit) + .map(|statements| statements.len()) + .map_err(|failure| failure.message) + }) + .expect("the small-stack thread starts") + .join() + .unwrap_or_else(|_| panic!("{}: the parse panicked", test_case.description)); + let ends_as_expected: Result = + outcome.as_ref().map(|count| *count).map_err(|message| { + message.ends_with(test_case.expected_outcome.err().unwrap_or_default()) + }); + assert_eq!( + ends_as_expected, + test_case.expected_outcome.map_err(|_| true), + "{}: {outcome:?}", + test_case.description + ); + } +} + +#[test] +fn given_a_requested_call_depth_when_checked_then_only_1_to_512_is_accepted() { + let test_cases = [ + CheckedDepthTestCase { + description: "zero is refused", + depth: 0, + expected_result: Err("the function-call depth limit must be between 1 and 512, \ + SQLQuack's fixed nesting ceiling, not 0"), + }, + CheckedDepthTestCase { + description: "one is the lowest limit", + depth: 1, + expected_result: Ok(1), + }, + CheckedDepthTestCase { + description: "the ceiling itself is accepted", + depth: 512, + expected_result: Ok(512), + }, + CheckedDepthTestCase { + description: "one past the ceiling is refused", + depth: 513, + expected_result: Err("the function-call depth limit must be between 1 and 512, \ + SQLQuack's fixed nesting ceiling, not 513"), + }, + ]; + + for test_case in test_cases { + assert_eq!( + checked_call_depth(test_case.depth), + test_case.expected_result.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlquack-parse/src/parsing/tests/test_types.rs b/crates/sqlquack-parse/src/parsing/tests/test_types.rs new file mode 100644 index 000000000..ed3c05bd8 --- /dev/null +++ b/crates/sqlquack-parse/src/parsing/tests/test_types.rs @@ -0,0 +1,14 @@ +pub(crate) struct CallDepthTestCase { + pub(crate) description: &'static str, + /// The caller's limit, or `None` for `parse_source`'s default. + pub(crate) limit: Option, + pub(crate) levels: usize, + /// How many statements parse, or how the parse error ends. + pub(crate) expected_outcome: Result, +} + +pub(crate) struct CheckedDepthTestCase { + pub(crate) description: &'static str, + pub(crate) depth: usize, + pub(crate) expected_result: Result, +} diff --git a/crates/sqlquack-python/Cargo.toml b/crates/sqlquack-python/Cargo.toml index fc8c25ecd..73f2ff977 100644 --- a/crates/sqlquack-python/Cargo.toml +++ b/crates/sqlquack-python/Cargo.toml @@ -27,6 +27,7 @@ serde.workspace = true serde_json.workspace = true sqlquack.workspace = true sqlquack-diagnostics.workspace = true +sqlquack-parse.workspace = true sqlquack-policy.workspace = true sqlquack-semantics = { workspace = true, features = ["snowflake"] } diff --git a/crates/sqlquack-python/src/bindings/_helpers/check_json.rs b/crates/sqlquack-python/src/bindings/_helpers/check_json.rs index 3cea73988..1fc739808 100644 --- a/crates/sqlquack-python/src/bindings/_helpers/check_json.rs +++ b/crates/sqlquack-python/src/bindings/_helpers/check_json.rs @@ -5,13 +5,19 @@ use pyo3::{PyResult, pyfunction}; use sqlquack::analysis::main::check_source::check_source; use sqlquack::cli::models::CheckReport; use sqlquack_diagnostics::diagnostics::main::render_diagnostics::render_diagnostics; +use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use crate::bindings::models::Rendered; -/// The check report of `sql` as JSON, naming the source `path`; `ValueError` without SQL. +/// The check report of `sql` as JSON, named `path`; `ValueError` without SQL or past the ceiling. #[pyfunction] -pub(crate) fn check_json(sql: &str, path: &str) -> PyResult { - let diagnostics = check_source(sql).map_err(PyValueError::new_err)?; +#[pyo3(signature = (sql, path, max_call_depth=None))] +pub(crate) fn check_json(sql: &str, path: &str, max_call_depth: Option) -> PyResult { + let max_call_depth = max_call_depth + .map(checked_call_depth) + .transpose() + .map_err(|reason| PyValueError::new_err(format!("max_call_depth: {reason}")))?; + let diagnostics = check_source(sql, max_call_depth).map_err(PyValueError::new_err)?; let rendered = render_diagnostics(&diagnostics, sql, path); let report = CheckReport { ok: diagnostics.is_empty(), diff --git a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs index 2f8176f8f..7853ea325 100644 --- a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs +++ b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs @@ -4,10 +4,12 @@ use pyo3::exceptions::PyValueError; use pyo3::{PyResult, pyfunction}; use sqlquack::cli::main::convert_report::convert_report; use sqlquack::cli::models::{ - ColorArguments, ConvertArguments, OutputArguments, OutputFormat, PolicyArguments, + ColorArguments, ConvertArguments, OutputArguments, OutputFormat, ParseArguments, + PolicyArguments, }; use sqlquack_diagnostics::diagnostics::main::render_diagnostics::render_diagnostics; use sqlquack_diagnostics::diagnostics::models::Diagnostic; +use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use sqlquack_semantics::snowflake::types::errors::SchemaError; use sqlquack_semantics::snowflake::types::main::parse_schema::parse_schema; @@ -49,6 +51,13 @@ pub(crate) fn convert_json(sql: &str, request: &str, path: &str) -> PyResult, /// `--compact`. pub compact: bool, + /// `--max-call-depth`; the default when absent. + pub max_call_depth: Option, } diff --git a/crates/sqlquack/src/analysis/main/check_source.rs b/crates/sqlquack/src/analysis/main/check_source.rs index 754d3de89..e8259ff26 100644 --- a/crates/sqlquack/src/analysis/main/check_source.rs +++ b/crates/sqlquack/src/analysis/main/check_source.rs @@ -4,14 +4,14 @@ use sqlquack_diagnostics::diagnostics::main::diagnose::diagnose; use sqlquack_diagnostics::diagnostics::models::Diagnostic; use sqlquack_diagnostics::refusals::main::refuse_unparsed::refuse_unparsed; use sqlquack_diagnostics::snowflake::refusals::main::snowflake_rejections::snowflake_rejections; -use sqlquack_parse::parsing::main::parse_source::parse_source; +use sqlquack_parse::parsing::main::parse_source_within::parse_source_within; use sqlquack_syntax::Dialect; use sqlquack_syntax::tokens::models::SourceLines; -/// The parse failure, or every construct Snowflake would reject; empty when the SQL is sound. -pub fn check_source(sql: &str) -> Result, String> { +/// The parse failure, or every construct Snowflake rejects, calls nesting `max_call_depth` deep. +pub fn check_source(sql: &str, max_call_depth: Option) -> Result, String> { let lines = SourceLines::new(sql); - let refusals = match parse_source(sql, Dialect::Snowflake) { + let refusals = match parse_source_within(sql, Dialect::Snowflake, max_call_depth) { Ok(statements) if statements.is_empty() => { return Err("the input holds no SQL statement".to_owned()); } diff --git a/crates/sqlquack/src/analysis/main/collect_all_refusals.rs b/crates/sqlquack/src/analysis/main/collect_all_refusals.rs index 4aed244af..18514a8da 100644 --- a/crates/sqlquack/src/analysis/main/collect_all_refusals.rs +++ b/crates/sqlquack/src/analysis/main/collect_all_refusals.rs @@ -10,7 +10,7 @@ use sqlquack_diagnostics::refusals::main::refuse_unparsed::refuse_unparsed; use sqlquack_diagnostics::refusals::models::RefusalCategory; use sqlquack_diagnostics::refusals::types::ConstructPolicy; use sqlquack_diagnostics::snowflake::refusals::main::snowflake_rejections::snowflake_rejections; -use sqlquack_parse::parsing::main::parse_source::parse_source; +use sqlquack_parse::parsing::main::parse_source_within::parse_source_within; use sqlquack_semantics::snowflake::typing::types::Schema; use sqlquack_syntax::tokens::models::SourceLines; @@ -28,7 +28,7 @@ pub fn collect_all_refusals( quoted_exclusions: quoted_exclusions(sql), ..options.clone() }; - let statements = match parse_source(sql, dialect.syntax()) { + let statements = match parse_source_within(sql, dialect.syntax(), options.max_call_depth) { Ok(statements) => statements, Err(failure) => { return CollectedRefusals { diff --git a/crates/sqlquack/src/analysis/tests/helpers.rs b/crates/sqlquack/src/analysis/tests/helpers.rs index cd1f521e1..90a5e153e 100644 --- a/crates/sqlquack/src/analysis/tests/helpers.rs +++ b/crates/sqlquack/src/analysis/tests/helpers.rs @@ -38,7 +38,8 @@ use crate::analysis::tests::test_types::{ }; use crate::cli::main::convert_report::convert_report; use crate::cli::models::{ - ColorArguments, ConvertArguments, OutputArguments, OutputFormat, PolicyArguments, + ColorArguments, ConvertArguments, OutputArguments, OutputFormat, ParseArguments, + PolicyArguments, }; use sqlquack_convert::snowflake::analysis::main::analyze::analyze; use sqlquack_convert::snowflake::analysis::main::analyze_function::analyze_function; @@ -947,13 +948,14 @@ fn everyday_arguments(root: &std::path::Path, no_config: bool) -> ConvertArgumen no_config, strict: false, python_runtime: false, + parse: ParseArguments::default(), }, } } /// What SQLQuack does with `sql`: its first diagnostic or refusal, with span, or `translated`. fn everyday_outcome(sql: &str, arguments: &ConvertArguments) -> (String, Option) { - let diagnostics = check_source(sql).expect("the query holds a statement"); + let diagnostics = check_source(sql, None).expect("the query holds a statement"); diagnostics.first().map_or_else( || converted_outcome(sql, arguments), |found| { diff --git a/crates/sqlquack/src/cli/_helpers/convert_command.rs b/crates/sqlquack/src/cli/_helpers/convert_command.rs index 345e7d34f..3556e6a49 100644 --- a/crates/sqlquack/src/cli/_helpers/convert_command.rs +++ b/crates/sqlquack/src/cli/_helpers/convert_command.rs @@ -486,18 +486,19 @@ pub(crate) fn check_command( ) -> ExitCode { let _ = writeln!(stderr, "sqlquack check: checking…"); let path = arguments.path.as_deref(); - let (sql, diagnostics) = - match read_sql(path).and_then(|sql| check_source(&sql).map(|found| (sql, found))) { - Ok(checked) => checked, - Err(error) => { - let _ = writeln!( - stderr, - "sqlquack check: {}failed{}: {error}", - palette.error, palette.reset - ); - return ExitCode::from(2); - } - }; + let (sql, diagnostics) = match read_sql(path).and_then(|sql| { + check_source(&sql, arguments.parse.max_call_depth).map(|found| (sql, found)) + }) { + Ok(checked) => checked, + Err(error) => { + let _ = writeln!( + stderr, + "sqlquack check: {}failed{}: {error}", + palette.error, palette.reset + ); + return ExitCode::from(2); + } + }; let report = CheckReport { ok: diagnostics.is_empty(), diagnostics, diff --git a/crates/sqlquack/src/cli/_helpers/evidence.rs b/crates/sqlquack/src/cli/_helpers/evidence.rs index bdf1017fc..3717b7c13 100644 --- a/crates/sqlquack/src/cli/_helpers/evidence.rs +++ b/crates/sqlquack/src/cli/_helpers/evidence.rs @@ -7,6 +7,7 @@ use sqlquack_convert::snowflake::conversion::models::ConversionOptions; use sqlquack_convert::snowflake::session::main::configured_options::configured_options; use sqlquack_evidence::cases::main::load_cases::load_cases; use sqlquack_evidence::cases::models::LoadedCase; +use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use sqlquack_policy::config::models::UserConfig; use sqlquack_policy::policy::main::load_policy::load_policy; use sqlquack_policy::policy::models::Policy; @@ -29,6 +30,15 @@ pub(crate) fn divergence_options(choices: &ConversionChoices) -> ConversionOptio configured_options(choices) } +/// Parses `--max-call-depth`, a whole number from 1 up to the parser's fixed nesting ceiling. +pub(crate) fn parse_call_depth(depth: &str) -> Result { + let depth: usize = depth + .trim() + .parse() + .map_err(|_| format!("--max-call-depth needs a whole number, not `{depth}`"))?; + checked_call_depth(depth) +} + /// Parses `10%` and similar into a percentage between 0 and 100. pub(crate) fn parse_percent(sample: &str) -> Result { let invalid = || format!("--sample needs a percentage such as 10%, not `{sample}`"); @@ -49,5 +59,6 @@ pub(crate) fn conversion_choices( allow: config.allow.clone(), strict: arguments.strict, python_runtime: arguments.python_runtime || config.project.python_runtime, + max_call_depth: arguments.parse.max_call_depth, } } diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs index ac4f24667..02e8750d3 100644 --- a/crates/sqlquack/src/cli/models.rs +++ b/crates/sqlquack/src/cli/models.rs @@ -22,6 +22,7 @@ use sqlquack_syntax::tokens::models::SourceLocation; use crate::analysis::models::PythonSignature; use crate::census::constants::DEFAULT_TEXT_LIMIT; use crate::census::models::CursorType; +use crate::cli::_helpers::evidence::parse_call_depth; use crate::cli::constants::EXIT_STATUS_HELP; /// Snowflake-to-DuckDB SQL conversion that refuses to guess. @@ -101,6 +102,9 @@ pub struct CheckArguments { /// How standard output and standard error are coloured. #[command(flatten)] pub colors: ColorArguments, + /// How deep the SQL may nest. + #[command(flatten)] + pub parse: ParseArguments, } /// Arguments of `sqlquack convert`. @@ -394,6 +398,17 @@ pub struct PolicyArguments { /// Confirm a Python runtime registers the project's Python functions with SQLQuack's runtime. #[arg(long)] pub python_runtime: bool, + /// How deep the SQL may nest. + #[command(flatten)] + pub parse: ParseArguments, +} + +/// How deep the SQL may nest. +#[derive(Clone, Debug, Default, Args)] +pub struct ParseArguments { + /// Refuse function calls nested deeper than this; 256 by default, at most 512. + #[arg(long, value_name = "DEPTH", value_parser = parse_call_depth)] + pub max_call_depth: Option, } /// How a report is printed. diff --git a/crates/sqlquack/tests/cli/test_check.rs b/crates/sqlquack/tests/cli/test_check.rs index 84bbf59ef..781903ef0 100644 --- a/crates/sqlquack/tests/cli/test_check.rs +++ b/crates/sqlquack/tests/cli/test_check.rs @@ -1,5 +1,7 @@ use crate::helpers::{first_refusal_help, run_with_environment, run_with_input}; -use crate::test_types::{CheckCommandTestCase, ColorTestCase, DivergenceHelpTestCase}; +use crate::test_types::{ + CallDepthCommandTestCase, CheckCommandTestCase, ColorTestCase, DivergenceHelpTestCase, +}; const PLAIN_CHECK: &str = "sqlquack check: checking…\n\ error[SQ0203]: QUALIFY needs a window function\n \ @@ -21,6 +23,117 @@ const PLAIN_CONVERTED: &str = "sqlquack convert: converting…\n\ const STYLED_CONVERTED: &str = "sqlquack convert: converting…\n\ sqlquack convert: \x1b[1;32mdone, converted\x1b[0m\n"; +#[test] +fn given_nested_function_calls_when_checked_or_converted_then_the_call_depth_limit_decides() { + let test_cases = [ + CallDepthCommandTestCase { + description: "check reads 256 nested calls by default", + arguments: &["check"], + levels: 256, + expected_exit_code: Some(0), + expected_stderr_mentions: &["sqlquack check: done, no problems"], + expected_stderr_absences: &[], + }, + CallDepthCommandTestCase { + description: "check refuses 257 nested calls, naming the limit and how to raise it", + arguments: &["check"], + levels: 257, + expected_exit_code: Some(1), + expected_stderr_mentions: &[ + "error[SQ0107]", + "value 257 exceeds configured limit 256", + "= help: compute inner calls in a common table expression first, or raise the \ + limit, up to 512, with --max-call-depth (command line) or max_call_depth= (Python)", + ], + expected_stderr_absences: &[], + }, + CallDepthCommandTestCase { + description: "check --max-call-depth 300 reads 300 nested calls", + arguments: &["check", "--max-call-depth", "300"], + levels: 300, + expected_exit_code: Some(0), + expected_stderr_mentions: &["sqlquack check: done, no problems"], + expected_stderr_absences: &[], + }, + CallDepthCommandTestCase { + description: "check --max-call-depth 1000 is refused, naming the ceiling", + arguments: &["check", "--max-call-depth", "1000"], + levels: 300, + expected_exit_code: Some(2), + expected_stderr_mentions: &[ + "must be between 1 and 512, SQLQuack's fixed nesting ceiling, not 1000", + ], + expected_stderr_absences: &[], + }, + CallDepthCommandTestCase { + description: "at the ceiling, 600 nested calls meet the fixed limit, which says so", + arguments: &["check", "--max-call-depth", "512"], + levels: 600, + expected_exit_code: Some(1), + expected_stderr_mentions: &[ + "exceeds configured limit 512; 512 is SQLQuack's fixed nesting ceiling, which no \ + setting raises", + "the limit of 512 is fixed, and --max-call-depth can't raise it", + ], + expected_stderr_absences: &["SQ0107"], + }, + CallDepthCommandTestCase { + description: "convert refuses 257 nested calls by default", + arguments: &["convert", "--no-config"], + levels: 257, + expected_exit_code: Some(1), + expected_stderr_mentions: &["error[SQ0107]", "--max-call-depth"], + expected_stderr_absences: &[], + }, + CallDepthCommandTestCase { + description: "convert --max-call-depth 300 parses 300 nested calls", + arguments: &["convert", "--no-config", "--max-call-depth", "300"], + levels: 300, + expected_exit_code: Some(0), + expected_stderr_mentions: &["sqlquack convert: done, converted"], + expected_stderr_absences: &["SQ0107"], + }, + CallDepthCommandTestCase { + description: "run takes the flag and refuses one past the ceiling", + arguments: &["run", "--no-config", "--max-call-depth", "513"], + levels: 1, + expected_exit_code: Some(2), + expected_stderr_mentions: &["SQLQuack's fixed nesting ceiling, not 513"], + expected_stderr_absences: &[], + }, + ]; + + for test_case in test_cases { + let directory = tempfile::tempdir().expect("a temporary folder"); + let sql = format!( + "SELECT {}1{} AS total", + "ABS(".repeat(test_case.levels), + ")".repeat(test_case.levels) + ); + let (exit_code, _stdout, stderr) = + run_with_input(directory.path(), test_case.arguments, &sql); + assert_eq!( + exit_code, test_case.expected_exit_code, + "{}: {stderr}", + test_case.description + ); + for mention in test_case.expected_stderr_mentions { + assert!( + stderr.contains(mention), + "{}: {mention:?} in {stderr}", + test_case.description + ); + } + for absence in test_case.expected_stderr_absences { + assert!( + !stderr.contains(absence), + "{}: {absence:?} in {stderr}", + test_case.description + ); + } + } +} + #[test] fn given_snowflake_sql_when_checked_then_problems_are_coded_and_the_exit_status_says_so() { let test_cases = [ diff --git a/crates/sqlquack/tests/cli/test_types.rs b/crates/sqlquack/tests/cli/test_types.rs index fd821af1b..b8ff54fec 100644 --- a/crates/sqlquack/tests/cli/test_types.rs +++ b/crates/sqlquack/tests/cli/test_types.rs @@ -268,6 +268,17 @@ pub(crate) struct CheckCommandTestCase { pub(crate) expected_stderr_mention: &'static str, } +pub(crate) struct CallDepthCommandTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + /// The SQL is a SELECT of this many nested ABS calls. + pub(crate) levels: usize, + pub(crate) expected_exit_code: Option, + pub(crate) expected_stderr_mentions: &'static [&'static str], + /// Text stderr must not hold, such as a code the case must not report. + pub(crate) expected_stderr_absences: &'static [&'static str], +} + pub(crate) struct FailureKindTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/python/src/sqlquack/_native.pyi b/python/src/sqlquack/_native.pyi index ee1ed3134..ab136a74a 100644 --- a/python/src/sqlquack/_native.pyi +++ b/python/src/sqlquack/_native.pyi @@ -2,7 +2,7 @@ __version__: str -def check_json(sql: str, path: str) -> str: +def check_json(sql: str, path: str, max_call_depth: int | None = None) -> str: """``{"report": , "rendered": }``; ``ValueError`` without SQL.""" def convert_json(sql: str, request: str, path: str) -> str: diff --git a/python/src/sqlquack/api/main/check.py b/python/src/sqlquack/api/main/check.py index d8231c747..7609ba02b 100644 --- a/python/src/sqlquack/api/main/check.py +++ b/python/src/sqlquack/api/main/check.py @@ -10,9 +10,9 @@ from sqlquack.api.models import CheckResult -def check(*, sql: str, path: str = DEFAULT_PATH) -> CheckResult: +def check(*, sql: str, path: str = DEFAULT_PATH, max_call_depth: int | None = None) -> CheckResult: """Parse `sql` and find what Snowflake rejects, naming it `path`; no SQL raises.""" - result: dict[str, Any] = loaded(lambda: check_json(sql, path)) + result: dict[str, Any] = loaded(lambda: check_json(sql, path, max_call_depth)) report: dict[str, Any] = result["report"] return CheckResult( ok=report["ok"], diff --git a/python/src/sqlquack/api/main/convert.py b/python/src/sqlquack/api/main/convert.py index 4966e3123..be7c39979 100644 --- a/python/src/sqlquack/api/main/convert.py +++ b/python/src/sqlquack/api/main/convert.py @@ -27,6 +27,7 @@ def convert( "overrides": path_argument(chosen.overrides), "verified_overrides": dict(chosen.verified_overrides), "compact": chosen.compact, + "max_call_depth": chosen.max_call_depth, } result: dict[str, Any] = loaded(lambda: convert_json(sql, json.dumps(request), path)) report: dict[str, Any] = result["report"] diff --git a/python/src/sqlquack/api/models.py b/python/src/sqlquack/api/models.py index feb142ba3..d27afe15a 100644 --- a/python/src/sqlquack/api/models.py +++ b/python/src/sqlquack/api/models.py @@ -75,3 +75,4 @@ class ConvertOptions: overrides: str | os.PathLike[str] | None = None verified_overrides: Mapping[str, str] = field(default_factory=dict) compact: bool = False + max_call_depth: int | None = None diff --git a/tests/unit/python/src/sqlquack/api/main/_test_types.py b/tests/unit/python/src/sqlquack/api/main/_test_types.py index 6cbc93d3e..11a3d2958 100644 --- a/tests/unit/python/src/sqlquack/api/main/_test_types.py +++ b/tests/unit/python/src/sqlquack/api/main/_test_types.py @@ -25,6 +25,23 @@ class RenderTestCase: expected_help: str | None +@dataclass(frozen=True) +class CallDepthTestCase: + description: str + levels: int + max_call_depth: int | None + expected_codes: tuple[str, ...] + expected_helps: tuple[str, ...] + expected_labels: tuple[str, ...] + + +@dataclass(frozen=True) +class CallDepthRefusalTestCase: + description: str + max_call_depth: int + expected_message: str + + @dataclass(frozen=True) class FailureTestCase: description: str diff --git a/tests/unit/python/src/sqlquack/api/main/test_check.py b/tests/unit/python/src/sqlquack/api/main/test_check.py index 683859a3d..2fd14609d 100644 --- a/tests/unit/python/src/sqlquack/api/main/test_check.py +++ b/tests/unit/python/src/sqlquack/api/main/test_check.py @@ -2,11 +2,15 @@ from __future__ import annotations +import threading + import pytest import sqlquack from sqlquack import CheckResult from tests.unit.python.src.sqlquack.api.main._test_types import ( + CallDepthRefusalTestCase, + CallDepthTestCase, CheckTestCase, FailureTestCase, RenderTestCase, @@ -95,3 +99,108 @@ def test_given_no_statement_when_checking_then_sqlquack_error_is_raised( ) -> None: with pytest.raises(sqlquack.SqlQuackError, match=test_case.expected_message): sqlquack.check(sql=test_case.sql) + + +SMALL_STACK_BYTES: int = 256 * 1024 +FIXED_NESTING_HELP: str = ( + "remove redundant parentheses, or move nested parts into common table expressions; " + "the limit of 512 is fixed, and --max-call-depth can't raise it" +) + + +@pytest.mark.parametrize( + "test_case", + ( + CallDepthTestCase( + description="256 nested calls are read by default", + levels=256, + max_call_depth=None, + expected_codes=(), + expected_helps=(), + expected_labels=(), + ), + CallDepthTestCase( + description="257 nested calls are refused, saying how to raise the limit", + levels=257, + max_call_depth=None, + expected_codes=("SQ0107",), + expected_helps=( + "compute inner calls in a common table expression first, or raise the limit, up " + "to 512, with --max-call-depth (command line) or max_call_depth= (Python)", + ), + expected_labels=("value 257 exceeds configured limit 256",), + ), + CallDepthTestCase( + description="max_call_depth=300 reads 300 nested calls", + levels=300, + max_call_depth=300, + expected_codes=(), + expected_helps=(), + expected_labels=(), + ), + CallDepthTestCase( + description="at the ceiling, 600 nested calls meet the fixed limit, which says so", + levels=600, + max_call_depth=512, + expected_codes=("SQ0103",), + expected_helps=(FIXED_NESTING_HELP,), + expected_labels=( + "value 513 exceeds configured limit 512; 512 is SQLQuack's fixed nesting ceiling, " + "which no setting raises", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_nested_calls_when_checked_on_a_small_stack_then_the_call_depth_limit_decides( + test_case: CallDepthTestCase, +) -> None: + sql: str = "SELECT " + "ABS(" * test_case.levels + "1" + ")" * test_case.levels + results: list[CheckResult] = [] + previous: int = threading.stack_size(SMALL_STACK_BYTES) + try: + thread: threading.Thread = threading.Thread( + target=lambda: results.append( + sqlquack.check(sql=sql, max_call_depth=test_case.max_call_depth) + ) + ) + thread.start() + thread.join() + finally: + threading.stack_size(previous) + (result,) = results + assert tuple(found.code for found in result.diagnostics) == test_case.expected_codes + assert tuple(found.help for found in result.diagnostics) == test_case.expected_helps + assert tuple(found.label for found in result.diagnostics) == test_case.expected_labels + + +@pytest.mark.parametrize( + "test_case", + ( + CallDepthRefusalTestCase( + description="zero is refused", + max_call_depth=0, + expected_message="max_call_depth: .*SQLQuack's fixed nesting ceiling, not 0", + ), + CallDepthRefusalTestCase( + description="one past the ceiling is refused", + max_call_depth=513, + expected_message="max_call_depth: .*SQLQuack's fixed nesting ceiling, not 513", + ), + CallDepthRefusalTestCase( + description="1000 is refused, naming the ceiling", + max_call_depth=1000, + expected_message="max_call_depth: .*SQLQuack's fixed nesting ceiling, not 1000", + ), + ), + ids=lambda case: case.description, +) +def test_given_a_call_depth_past_the_ceiling_when_checking_or_converting_then_it_is_refused( + test_case: CallDepthRefusalTestCase, +) -> None: + with pytest.raises(sqlquack.SqlQuackError, match=test_case.expected_message): + sqlquack.check(sql="SELECT 1", max_call_depth=test_case.max_call_depth) + with pytest.raises(sqlquack.SqlQuackError, match=test_case.expected_message): + sqlquack.convert( + sql="SELECT 1", options=sqlquack.ConvertOptions(max_call_depth=test_case.max_call_depth) + ) diff --git a/website/src/content/docs/docs/cli/check.mdx b/website/src/content/docs/docs/cli/check.mdx index 02fe80d8b..b6c47ee8e 100644 --- a/website/src/content/docs/docs/cli/check.mdx +++ b/website/src/content/docs/docs/cli/check.mdx @@ -49,6 +49,7 @@ sqlquack check: found 1 problem | `--format text\|json` | Print the report as text for people or JSON for programs. Default `text`. | | `--json` | The same as `--format json`. | | `--color auto\|always\|never` | Colour diagnostics. Default `auto`: only on a terminal, honouring `NO_COLOR` and `CLICOLOR_FORCE`. | +| `--max-call-depth ` | Refuse function calls nested more than `n` deep with [`SQ0107`](/docs/diagnostics/reference/#sq0107). Default `256`, at most `512`, the parser's fixed nesting limit. | With `--json`, `check` prints `{"ok": …, "diagnostics": [...]}`. See [JSON output](/docs/diagnostics/overview/#json-output). @@ -59,4 +60,4 @@ With `--json`, `check` prints `{"ok": …, "diagnostics": [...]}`. See |------|---------| | `0` | No problem found. | | `1` | At least one problem found. | -| `2` | The input holds no SQL, or can't be read. | +| `2` | The input holds no SQL, can't be read, or an option is invalid, such as `--max-call-depth` above `512`. | diff --git a/website/src/content/docs/docs/cli/convert.mdx b/website/src/content/docs/docs/cli/convert.mdx index 8f190afff..b3457aea8 100644 --- a/website/src/content/docs/docs/cli/convert.mdx +++ b/website/src/content/docs/docs/cli/convert.mdx @@ -64,6 +64,7 @@ SQL that does not parse, or that Snowflake rejects, is refused with the same cod | `--overrides ` | [Replacements](/docs/policy/#replace): a `FUNCTION()` file, or a directory of them, each with `replaces` and `dialect duckdb`. Replaces the project's `overrides/` folder. | | `--verified-override NAME=SHA256` | An override verified for the body with that hash, as `convert --json` reports. | | `--compact` | Print each statement on one line instead, in the JSON report too. | +| `--max-call-depth ` | Refuse function calls nested more than `n` deep, as for [`check`](/docs/cli/check/#options). Default `256`, at most `512`. `run`, `function` and `census` take it too. | | `--format text\|json`, `--json` | Print the report as text or JSON. | | `--color auto\|always\|never` | Highlight the SQL and colour diagnostics, as for [`check`](/docs/cli/check/#options). JSON is never coloured. | diff --git a/website/src/content/docs/docs/diagnostics/reference.mdx b/website/src/content/docs/docs/diagnostics/reference.mdx index aad930114..1d6e4031f 100644 --- a/website/src/content/docs/docs/diagnostics/reference.mdx +++ b/website/src/content/docs/docs/diagnostics/reference.mdx @@ -193,9 +193,9 @@ Also reported by its guard name, `E_GUARD_TOKEN_BUDGET_EXCEEDED`. **brackets nest deeper than the parser accepts** (`nesting_depth_exceeded`) -Parentheses, brackets and braces nest deeper at the span than the parser's limit. The limit keeps parsing from exhausting the stack. Redundant parentheses can be removed; deeply nested subqueries can usually become common table expressions. +Parentheses, brackets and braces nest deeper at the span than the parser's limit of 512. The limit keeps parsing from exhausting the stack, and no setting raises it. Redundant parentheses can be removed; deeply nested subqueries can usually become common table expressions. -**Help:** remove redundant parentheses, or move nested parts into common table expressions +**Help:** remove redundant parentheses, or move nested parts into common table expressions; the limit of 512 is fixed, and --max-call-depth can't raise it **Example:** @@ -225,9 +225,9 @@ Also reported by its guard name, `E_GUARD_PARSER_DEPTH_EXCEEDED`. **the parsed statement nests deeper than SQLQuack accepts** (`ast_depth_exceeded`) -The statement parsed, but its tree nests deeper than the limit SQLQuack places on later analysis, which keeps conversion from exhausting the stack. Restructure the deepest part of the statement. +The statement parsed, but its tree nests deeper than the limit of 512 SQLQuack places on later analysis, which keeps conversion from exhausting the stack. No setting raises it. Restructure the deepest part of the statement. -**Help:** move nested parts into common table expressions +**Help:** move nested parts into common table expressions; the limit of 512 is fixed, and --max-call-depth can't raise it **Example:** @@ -253,9 +253,9 @@ Also reported by its guard name, `E_GUARD_AST_BUDGET_EXCEEDED`. **function calls nest deeper than SQLQuack accepts** (`function_nesting_depth_exceeded`) -Function calls nest inside each other deeper than SQLQuack's limit at the span. Compute the innermost calls in a common table expression or a subquery and refer to their results. +Function calls nest inside each other deeper than SQLQuack's limit at the span, 256 by default. Compute the innermost calls in a common table expression or a subquery and refer to their results, or raise the limit with `--max-call-depth` on the command line or `max_call_depth=` in Python. The limit goes up to 512, the parser's fixed nesting limit. -**Help:** compute inner calls in a common table expression first +**Help:** compute inner calls in a common table expression first, or raise the limit, up to 512, with --max-call-depth (command line) or max_call_depth= (Python) **Example:** diff --git a/website/src/content/docs/docs/python.mdx b/website/src/content/docs/docs/python.mdx index 2ffcb3c89..a190b7c9d 100644 --- a/website/src/content/docs/docs/python.mdx +++ b/website/src/content/docs/docs/python.mdx @@ -45,8 +45,10 @@ With an existing Python 3.12 or newer, `pip install target/wheels/sqlquack-*.whl ## Checking -`check(*, sql, path="")` parses `sql` and finds what Snowflake rejects. `path` names the -SQL in rendered diagnostics. It returns a `CheckResult`: +`check(*, sql, path="", max_call_depth=None)` parses `sql` and finds what Snowflake +rejects. `path` names the SQL in rendered diagnostics. `max_call_depth` is +[`--max-call-depth`](/docs/cli/check/#options): function calls may nest that deep, `256` by +default and at most `512`. It returns a `CheckResult`: | Field | Type | Meaning | |-------|------|---------| @@ -75,7 +77,7 @@ SQL in rendered diagnostics. It returns a `CheckResult`: `ConvertOptions` has one field per [`convert`](/docs/cli/convert/) flag, with the same defaults: `project`, `config`, `no_config`, `session`, `strict`, `python_runtime`, `overrides`, -`verified_overrides` and `compact`. `project`, `config` and `overrides` take a path (`str` or +`verified_overrides`, `compact` and `max_call_depth`. `project`, `config` and `overrides` take a path (`str` or `os.PathLike`), not the parsed contents. `session` and `verified_overrides` take a mapping, such as `session={"TIMEZONE": "UTC"}`, not a list of `NAME=VALUE` strings. @@ -101,4 +103,5 @@ SQL. ## Errors Where the command line exits 2, the functions raise `SqlQuackError`, a `ValueError`: for example, -for input that holds no SQL statement or a `schema` that is not a valid mapping. +for input that holds no SQL statement, a `schema` that is not a valid mapping, or a +`max_call_depth` above `512`.