Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/build-distributions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,20 @@ jobs:
"${python[@]}" -c 'import sqlquack
result = sqlquack.convert(sql="SELECT 1 AS total", options=sqlquack.ConvertOptions(no_config=True))
assert result.sql == "SELECT CAST(1 AS DECIMAL(1, 0)) AS \"TOTAL\"", result.sql'
# Deep function-call nesting on a small thread stack, where Windows threads are smallest:
# 511 levels read at the 512 ceiling; 257 at the default is a clean error, not a crash.
"${python[@]}" -c 'import sqlquack, threading
def nested(levels):
return "SELECT " + "ABS(" * levels + "1" + ")" * levels
found = []
threading.stack_size(256 * 1024)
def check():
found.append(sqlquack.check(sql=nested(511), max_call_depth=512).ok)
found.append(sqlquack.check(sql=nested(257)).diagnostics[0].code)
thread = threading.Thread(target=check)
thread.start()
thread.join()
assert found == [True, "SQ0107"], found'
# A transient network error in the upload would otherwise cost the whole build, so it retries
# once, replacing any artifact the failed attempt left behind.
- id: upload-wheel
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! `analyze`: parse, classify, convert or refuse.

use sqlquack_diagnostics::classification::main::classify::classify;
use sqlquack_parse::parsing::main::parse_source::parse_source;
use sqlquack_parse::parsing::main::parse_source_within::parse_source_within;
use sqlquack_syntax::Expression;
use std::time::Instant;

Expand Down Expand Up @@ -36,7 +36,7 @@ pub fn analyze(
quoted_exclusions: quoted_exclusions(sql),
..options.clone()
};
let statements = match parse_source(sql, Dialect::Snowflake) {
let statements = match parse_source_within(sql, Dialect::Snowflake, options.max_call_depth) {
Ok(statements) => statements,
Err(failure) => {
let refusal = refuse_unparsed(&failure, &lines);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use sqlquack_diagnostics::refusals::main::refuse_unparsed::refuse_unparsed;
use sqlquack_diagnostics::refusals::models::{Refusal, RefusalCategory};
use sqlquack_diagnostics::refusals::types::ConstructPolicy;
use sqlquack_diagnostics::snowflake::classification::main::cast_type_name::cast_type_name;
use sqlquack_parse::parsing::main::parse_source::parse_source;
use sqlquack_parse::parsing::main::parse_source_within::parse_source_within;
use sqlquack_semantics::snowflake::typing::models::SnowflakeType;
use sqlquack_semantics::snowflake::typing::types::{FunctionReturns, FunctionSpec, Schema};
use sqlquack_syntax::Dialect;
Expand Down Expand Up @@ -42,7 +42,7 @@ pub fn analyze_function(
..options.clone()
};
let lines = SourceLines::shifted(body, prefix.chars().count());
let statements = parse_source(&sql, Dialect::Snowflake)
let statements = parse_source_within(&sql, Dialect::Snowflake, options.max_call_depth)
.map_err(|failure| vec![refuse_unparsed(&failure, &lines)])?;
let [statement] = statements.as_slice() else {
return Err(vec![Refusal {
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlquack-convert/src/snowflake/analysis/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,6 @@ pub struct ConversionChoices {
pub strict: bool,
/// A Python runtime registers the project's Python functions.
pub python_runtime: bool,
/// Function calls nested deeper than this are a parse error; the parser's default when absent.
pub max_call_depth: Option<usize>,
}
2 changes: 2 additions & 0 deletions crates/sqlquack-convert/src/snowflake/conversion/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ pub struct ConversionOptions {
pub overrides: BTreeMap<String, UserOverride>,
/// Override names to the SHA-256 of the body recorded cases verified, which strict mode allows.
pub verified_overrides: BTreeMap<String, String>,
/// Function calls nested deeper than this are a parse error; the parser's default when absent.
pub max_call_depth: Option<usize>,
}

/// A user's DuckDB SQL standing in for a Snowflake function, written in SQLBuild's FUNCTION() format.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ pub fn configured_choices(
allow: config.allow,
strict,
python_runtime: config.project.python_runtime,
max_call_depth: None,
};
Ok((choices, session))
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use sqlquack_policy::policy::main::evidence_policy::evidence_policy;
pub fn configured_options(choices: &ConversionChoices) -> ConversionOptions {
let evidence = evidence_policy();
let mut options = builtin_conversion_options(choices.strict);
options.max_call_depth = choices.max_call_depth;
if let Some(budget) = evidence.conversion_budget {
options.budget = ConversionBudget {
milliseconds: budget.milliseconds,
Expand Down
31 changes: 20 additions & 11 deletions crates/sqlquack-diagnostics/src/diagnostics/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,12 +261,13 @@ pub const CODES: &[CodeEntry] = &[
alias: Some("E_GUARD_NESTING_DEPTH_EXCEEDED"),
title: "brackets nest deeper than the parser accepts",
help: Some(
"remove redundant parentheses, or move nested parts into common table expressions",
"remove redundant parentheses, or move nested parts into common table expressions; \
the limit of 512 is fixed, and --max-call-depth can't raise it",
),
explanation: "Parentheses, brackets and braces nest deeper at the span than the \
parser's limit. The limit keeps parsing from exhausting the stack. Redundant \
parentheses can be removed; deeply nested subqueries can usually become common \
table expressions.",
parser's limit of 512. The limit keeps parsing from exhausting the stack, and no \
setting raises it. Redundant parentheses can be removed; deeply nested subqueries \
can usually become common table expressions.",
example: r#"python3 -c "print('SELECT ' + '(' * 600 + '1' + ')' * 600)" | sqlquack check"#,
},
CodeEntry {
Expand All @@ -285,10 +286,13 @@ pub const CODES: &[CodeEntry] = &[
name: "ast_depth_exceeded",
alias: Some("E_GUARD_AST_DEPTH_EXCEEDED"),
title: "the parsed statement nests deeper than SQLQuack accepts",
help: Some("move nested parts into common table expressions"),
explanation: "The statement parsed, but its tree nests deeper than the limit SQLQuack \
places on later analysis, which keeps conversion from exhausting the stack. \
Restructure the deepest part of the statement.",
help: Some(
"move nested parts into common table expressions; the limit of 512 is fixed, and \
--max-call-depth can't raise it",
),
explanation: "The statement parsed, but its tree nests deeper than the limit of 512 \
SQLQuack places on later analysis, which keeps conversion from exhausting the stack. \
No setting raises it. Restructure the deepest part of the statement.",
example: r#"python3 -c "print('SELECT ' + 'CASE WHEN TRUE THEN ' * 600 + '1' + ' END' * 600)" | sqlquack check"#,
},
CodeEntry {
Expand All @@ -306,10 +310,15 @@ pub const CODES: &[CodeEntry] = &[
name: "function_nesting_depth_exceeded",
alias: Some("E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED"),
title: "function calls nest deeper than SQLQuack accepts",
help: Some("compute inner calls in a common table expression first"),
help: Some(
"compute inner calls in a common table expression first, or raise the limit, up to \
512, with --max-call-depth (command line) or max_call_depth= (Python)",
),
explanation: "Function calls nest inside each other deeper than SQLQuack's limit at \
the span. Compute the innermost calls in a common table expression or a subquery \
and refer to their results.",
the span, 256 by default. Compute the innermost calls in a common table expression \
or a subquery and refer to their results, or raise the limit with \
`--max-call-depth` on the command line or `max_call_depth=` in Python. The limit \
goes up to 512, the parser's fixed nesting limit.",
example: r#"python3 -c "print('SELECT ' + 'ABS(' * 300 + '1' + ')' * 300)" | sqlquack check"#,
},
CodeEntry {
Expand Down
11 changes: 7 additions & 4 deletions crates/sqlquack-lint/src/lint/_helpers/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -387,10 +387,13 @@ pub(crate) fn lint(request: LintRequest) -> Result<LintResponse, String> {
} = parse(
&request.sql,
dialect,
request
.fixture_harness
.as_ref()
.and_then(|harness| harness.empty_relation.as_deref()),
(
request
.fixture_harness
.as_ref()
.and_then(|harness| harness.empty_relation.as_deref()),
request.max_function_call_depth,
),
)?;
let external_identifiers: HashSet<String> = request
.external_identifiers
Expand Down
17 changes: 12 additions & 5 deletions crates/sqlquack-lint/src/lint/_helpers/parsing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,30 @@

use std::collections::HashSet;

use sqlquack_parse::parsing::constants::MAX_FUNCTION_CALL_DEPTH;
use sqlquack_parse::parsing::main::call_depth_hint::call_depth_hint;
use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth;
use sqlquack_syntax::tokens::models::{Token, TokenType};
use sqlquack_syntax::{ComplexityGuardOptions, Expression, ExpressionWalk, ParseOptions};

use crate::dialects::main::profile::profile;
use crate::dialects::models::LintProfile;
use crate::lint::models::{LintDialect, ParsedSql};

const MAX_LINT_FUNCTION_CALL_DEPTH: usize = 128;

/// Tokenize and parse `sql`, reading a harness's `FROM <empty_relation>()` as a bare relation.
pub(crate) fn parse(
sql: &str,
dialect: LintDialect,
empty_relation: Option<&str>,
(empty_relation, max_call_depth): (Option<&str>, Option<usize>),
) -> Result<ParsedSql, String> {
let max_call_depth: usize =
checked_call_depth(max_call_depth.unwrap_or(MAX_FUNCTION_CALL_DEPTH))
.map_err(|reason| format!("max_function_call_depth: {reason}"))?;
let profile: LintProfile = profile(dialect);
let tokens: Vec<Token> = (profile.tokens)(sql)?;
let options = ParseOptions {
complexity_guard: Some(ComplexityGuardOptions {
max_function_call_depth: Some(MAX_LINT_FUNCTION_CALL_DEPTH),
max_function_call_depth: Some(max_call_depth),
..Default::default()
}),
};
Expand All @@ -33,7 +37,10 @@ pub(crate) fn parse(
} else {
&parse_text
};
let statements: Vec<Expression> = (profile.statements)(parse_text, &options)?;
let statements: Vec<Expression> =
(profile.statements)(parse_text, &options).map_err(|message| {
call_depth_hint(message, max_call_depth, Some("max_function_call_depth"))
})?;
for statement in &statements {
if statement
.dfs()
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlquack-lint/src/lint/main/parse_sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ pub fn parse_sql(sql: &str, dialect: &str) -> Result<ParsedSql, String> {
let Some(parsed) = crate::dialects::main::lint_dialect::lint_dialect(dialect)? else {
return Err(format!("SQLQuack does not parse the {dialect} dialect yet"));
};
crate::lint::_helpers::parsing::parse(sql, parsed, None)
crate::lint::_helpers::parsing::parse(sql, parsed, (None, None))
}
3 changes: 3 additions & 0 deletions crates/sqlquack-lint/src/lint/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,9 @@ pub struct LintRequest {
pub max_literal_length: Option<usize>,
#[serde(default)]
pub max_ranking_order_by: Option<usize>,
/// Function calls nested deeper than this are a parse error; 256 when absent, at most 512.
#[serde(default)]
pub max_function_call_depth: Option<usize>,
#[serde(default)]
pub relation_keys: HashMap<String, Vec<Vec<String>>>,
#[serde(default)]
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlquack-lint/src/lint/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ mod batch_lint;
mod cross_joins;
#[path = "tests/test_empty_fixture_lint.rs"]
mod empty_fixture_lint;
#[path = "tests/test_function_depth.rs"]
mod function_depth;
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_join_predicates.rs"]
Expand Down
23 changes: 15 additions & 8 deletions crates/sqlquack-lint/src/lint/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,21 @@ pub(crate) fn nested_from_sql(depth: usize) -> String {
}

pub(crate) fn diagnostics(sql: &str) -> Result<Vec<Value>, String> {
let response = lint_json(
&json!({
"version": 1,
"sql": sql,
"dialect": "snowflake"
})
.to_string(),
)?;
diagnostics_within(sql, None)
}

/// The Snowflake diagnostics of `sql`, with the request's `max_function_call_depth`, null when absent.
pub(crate) fn diagnostics_within(
sql: &str,
max_function_call_depth: Option<usize>,
) -> Result<Vec<Value>, String> {
let request = json!({
"version": 1,
"sql": sql,
"dialect": "snowflake",
"max_function_call_depth": max_function_call_depth
});
let response = lint_json(&request.to_string())?;
diagnostic_values(&response)
}

Expand Down
82 changes: 82 additions & 0 deletions crates/sqlquack-lint/src/lint/tests/test_function_depth.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
use crate::lint::tests::{helpers, test_types};

#[test]
fn given_production_function_depth_when_linting_then_bounded_parser_accepts_it() {
let test_cases = [
test_types::FunctionDepthTestCase {
description: "production depth remains supported",
depth: 65,
max_function_call_depth: None,
expected_diagnostic_count: 0,
},
test_types::FunctionDepthTestCase {
description: "the default limit reads 256 nested calls",
depth: 256,
max_function_call_depth: None,
expected_diagnostic_count: 0,
},
test_types::FunctionDepthTestCase {
description: "a raised limit of 300 reads 300 nested calls",
depth: 300,
max_function_call_depth: Some(300),
expected_diagnostic_count: 0,
},
];
for test_case in test_cases {
let sql = helpers::nested_function_sql(test_case.depth);
let diagnostics = helpers::diagnostics_within(&sql, test_case.max_function_call_depth)
.unwrap_or_else(|error| {
panic!("{}: {error}", test_case.description);
});
assert_eq!(
diagnostics.len(),
test_case.expected_diagnostic_count,
"{}",
test_case.description
);
}
}

#[test]
fn given_excessive_function_depth_when_linting_then_complexity_guard_rejects_it() {
let test_cases = [
test_types::FunctionDepthFailureTestCase {
description: "the default limit refuses 257 nested calls and says how to raise it",
depth: 257,
max_function_call_depth: None,
expected_error_end: "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED: value 257 exceeds \
configured limit 256; raise max_function_call_depth, up to 512",
},
test_types::FunctionDepthFailureTestCase {
description: "a lowered limit of 10 refuses 11 nested calls",
depth: 11,
max_function_call_depth: Some(10),
expected_error_end: "value 11 exceeds configured limit 10; raise \
max_function_call_depth, up to 512",
},
test_types::FunctionDepthFailureTestCase {
description: "a limit past the ceiling is refused, naming the ceiling",
depth: 300,
max_function_call_depth: Some(1000),
expected_error_end: "max_function_call_depth: the function-call depth limit must be \
between 1 and 512, SQLQuack's fixed nesting ceiling, not 1000",
},
test_types::FunctionDepthFailureTestCase {
description: "at the ceiling of 512, 600 nested calls are refused, naming it as fixed",
depth: 600,
max_function_call_depth: Some(512),
expected_error_end: "E_GUARD_NESTING_DEPTH_EXCEEDED: value 513 exceeds configured \
limit 512; 512 is SQLQuack's fixed nesting ceiling, which no setting raises",
},
];
for test_case in test_cases {
let sql = helpers::nested_function_sql(test_case.depth);
let error = helpers::diagnostics_within(&sql, test_case.max_function_call_depth)
.expect_err(test_case.description);
assert!(
error.ends_with(test_case.expected_error_end),
"{}: {error}",
test_case.description
);
}
}
Loading
Loading