Skip to content

Commit 8dacf52

Browse files
committed
fix: drop every node a parser chains in a loop without recursion
A BigQuery pipe chain that ends in a syntax error is dropped on the error path before any depth guard runs, and PipeOperator's derived drop recursed once per step, so 20,000 steps aborted a 2 MiB debug thread. drop_iteratively! now covers PipeOperator and the other nodes a parser wraps around its own result in a loop: AT TIME ZONE, ** (BinaryFunc), XOR, the Snowflake ->> flow operator, model attributes, .: casts, array slices and method calls. Paren and Annotated stay derived: the parser moves out of them, the token guard bounds parentheses, and an annotation wraps one operand once. The BigQuery parser's with_ctes moved the input out of a PipeOperator, which a Drop impl forbids; it now walks to the chain's first input and replaces it in place, without recursion. A test builds a 100,000-step chain of each newly covered kind and drops it on a 2 MiB thread; it aborts without this change.
1 parent 897c4f5 commit 8dacf52

4 files changed

Lines changed: 171 additions & 2 deletions

File tree

‎crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/queries.rs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
33
use sqlquack_syntax::Expression;
44
use sqlquack_syntax::expressions::names::models::Identifier;
5+
use sqlquack_syntax::expressions::names::models::Null;
56
use sqlquack_syntax::expressions::operators::models::Alias;
67
use sqlquack_syntax::expressions::queries::models::{
78
ClusterBy, Cte, DistributeBy, From, GroupBy, Having, Limit, NamedWindow, OrderBy, Ordered,
@@ -237,7 +238,12 @@ fn reorganized(body: Expression, organization: Organization) -> Expression {
237238
/// `body` with `with` as its common table expressions, wrapped when it has its own already.
238239
pub(crate) fn with_ctes(body: Expression, with: With) -> Expression {
239240
if let Expression::PipeOperator(mut node) = body {
240-
node.this = with_ctes(node.this, with);
241+
let mut input = &mut node.this;
242+
while let Expression::PipeOperator(inner) = input {
243+
input = &mut inner.this;
244+
}
245+
let first = std::mem::replace(input, Expression::Null(Null));
246+
*input = with_ctes(first, with);
241247
return Expression::PipeOperator(node);
242248
}
243249
let has_with: bool = match &body {

‎crates/sqlquack-syntax/src/expressions/tree/_helpers/drops.rs‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
1-
//! Iterative drops for nodes that the parser chains in a loop, such as `x::INT::INT` or `v[1][1]`.
1+
//! Iterative drops for nodes a parser chains in a loop, such as `x::INT::INT`, `v[1][1]` or `q |> WHERE a |> WHERE b`.
22
3+
use crate::expressions::functions::models::JSONCast;
4+
use crate::expressions::functions::models::Operator;
5+
use crate::expressions::functions::models::Xor;
36
use crate::expressions::operators::models::Alias;
7+
use crate::expressions::operators::models::ArraySlice;
48
use crate::expressions::operators::models::Between;
59
use crate::expressions::operators::models::BinaryOp;
610
use crate::expressions::operators::models::Cast;
@@ -10,10 +14,15 @@ use crate::expressions::operators::models::In;
1014
use crate::expressions::operators::models::IsNull;
1115
use crate::expressions::operators::models::IsTrueFalse;
1216
use crate::expressions::operators::models::LikeOp;
17+
use crate::expressions::operators::models::MethodCall;
1318
use crate::expressions::operators::models::Subscript;
1419
use crate::expressions::operators::models::UnaryOp;
20+
use crate::expressions::queries::models::PipeOperator;
1521
use crate::expressions::tree::models::Expression;
22+
use crate::expressions::typed_functions::models::AtTimeZone;
23+
use crate::expressions::typed_functions::models::BinaryFunc;
1624
use crate::expressions::typed_functions::models::JsonExtractFunc;
25+
use crate::expressions::typed_functions::models::ModelAttribute;
1726
use crate::traversal::types::AstNode;
1827

1928
/// Frees a node's expression descendants with a worklist, so dropping a long chain cannot overflow the stack.
@@ -39,16 +48,25 @@ macro_rules! drop_iteratively {
3948

4049
drop_iteratively!(
4150
Alias,
51+
ArraySlice,
52+
AtTimeZone,
4253
Between,
54+
BinaryFunc,
4355
BinaryOp,
4456
Cast,
4557
CollationExpr,
4658
DotAccess,
4759
In,
4860
IsNull,
4961
IsTrueFalse,
62+
JSONCast,
5063
JsonExtractFunc,
5164
LikeOp,
65+
MethodCall,
66+
ModelAttribute,
67+
Operator,
68+
PipeOperator,
5269
Subscript,
5370
UnaryOp,
71+
Xor,
5472
);

‎crates/sqlquack-syntax/src/guard/tests/test_ast_guard.rs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,26 @@
1+
use crate::expressions::functions::models::JSONCast;
2+
use crate::expressions::functions::models::Operator;
3+
use crate::expressions::functions::models::Xor;
4+
use crate::expressions::names::models::Identifier;
5+
use crate::expressions::operators::models::ArraySlice;
6+
use crate::expressions::operators::models::MethodCall;
7+
use crate::expressions::queries::models::PipeOperator;
8+
use crate::expressions::tree::models::Expression;
9+
use crate::expressions::typed_functions::models::AtTimeZone;
10+
use crate::expressions::typed_functions::models::BinaryFunc;
11+
use crate::expressions::typed_functions::models::ModelAttribute;
12+
use crate::expressions::types::models::DataType;
113
use crate::guard::tests::helpers::constructor_levels;
214
use crate::guard::tests::helpers::debug_length;
315
use crate::guard::tests::helpers::deep_trees;
416
use crate::guard::tests::helpers::guarded_parse;
517
use crate::guard::tests::helpers::guarded_parse_in;
618
use crate::guard::tests::test_types::AstDepthTestCase;
19+
use crate::guard::tests::test_types::ChainStepTestCase;
720
use crate::guard::tests::test_types::TreeOperationTestCase;
821
use crate::guard::tests::test_types::TypeDepthTestCase;
922
use crate::guard::tests::test_types::TYPE_DEPTHS;
23+
use crate::traversal::types::ExpressionWalk;
1024

1125
#[test]
1226
fn given_a_long_flat_operator_chain_when_parsing_then_only_real_nesting_counts_as_depth() {
@@ -328,3 +342,125 @@ fn given_long_chains_and_deep_nesting_on_a_2_mb_thread_when_cloning_comparing_fo
328342
);
329343
}
330344
}
345+
346+
#[test]
347+
fn given_a_100_000_step_chain_of_a_node_parsers_build_in_a_loop_when_dropping_on_a_2_mb_thread_then_it_does_not_overflow(
348+
) {
349+
let test_cases = [
350+
ChainStepTestCase {
351+
description: "BigQuery pipe operators",
352+
step: |input| {
353+
Expression::PipeOperator(Box::new(PipeOperator {
354+
this: input,
355+
expression: Expression::number(1),
356+
}))
357+
},
358+
expected_depth: 100_000,
359+
},
360+
ChainStepTestCase {
361+
description: "AT TIME ZONE",
362+
step: |input| {
363+
Expression::AtTimeZone(Box::new(AtTimeZone {
364+
this: input,
365+
zone: Expression::number(1),
366+
}))
367+
},
368+
expected_depth: 100_000,
369+
},
370+
ChainStepTestCase {
371+
description: "** powers",
372+
step: |input| {
373+
Expression::Power(Box::new(BinaryFunc {
374+
this: input,
375+
expression: Expression::number(1),
376+
original_name: None,
377+
inferred_type: None,
378+
}))
379+
},
380+
expected_depth: 100_000,
381+
},
382+
ChainStepTestCase {
383+
description: "XOR",
384+
step: |input| {
385+
Expression::Xor(Box::new(Xor {
386+
this: Some(Box::new(input)),
387+
expression: Some(Box::new(Expression::number(1))),
388+
expressions: Vec::new(),
389+
}))
390+
},
391+
expected_depth: 100_000,
392+
},
393+
ChainStepTestCase {
394+
description: "Snowflake ->> flow operators",
395+
step: |input| {
396+
Expression::Operator(Box::new(Operator {
397+
this: Box::new(input),
398+
operator: None,
399+
expression: Box::new(Expression::number(1)),
400+
comments: Vec::new(),
401+
}))
402+
},
403+
expected_depth: 100_000,
404+
},
405+
ChainStepTestCase {
406+
description: "model attributes",
407+
step: |input| {
408+
Expression::ModelAttribute(Box::new(ModelAttribute {
409+
this: Box::new(input),
410+
expression: Box::new(Expression::number(1)),
411+
}))
412+
},
413+
expected_depth: 100_000,
414+
},
415+
ChainStepTestCase {
416+
description: ".: casts",
417+
step: |input| {
418+
Expression::JSONCast(Box::new(JSONCast {
419+
this: Box::new(input),
420+
to: DataType::Json,
421+
}))
422+
},
423+
expected_depth: 100_000,
424+
},
425+
ChainStepTestCase {
426+
description: "array slices",
427+
step: |input| {
428+
Expression::ArraySlice(Box::new(ArraySlice {
429+
this: input,
430+
start: Some(Expression::number(1)),
431+
end: None,
432+
}))
433+
},
434+
expected_depth: 100_000,
435+
},
436+
ChainStepTestCase {
437+
description: "method calls",
438+
step: |input| {
439+
Expression::MethodCall(Box::new(MethodCall {
440+
this: input,
441+
method: Identifier::new("f"),
442+
args: vec![Expression::number(1)],
443+
}))
444+
},
445+
expected_depth: 100_000,
446+
},
447+
];
448+
for test_case in test_cases {
449+
let step = test_case.step;
450+
let depth = std::thread::Builder::new()
451+
.stack_size(2 << 20)
452+
.spawn(move || {
453+
let mut chain = Expression::number(1);
454+
for _ in 0..100_000 {
455+
chain = step(chain);
456+
}
457+
let depth = chain.tree_depth();
458+
drop(chain);
459+
depth
460+
})
461+
.expect("the thread starts")
462+
.join()
463+
.expect("the chain drops");
464+
assert_eq!(depth, test_case.expected_depth, "{}", test_case.description);
465+
}
466+
}

‎crates/sqlquack-syntax/src/guard/tests/test_types.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
use serde_json::error::Category;
22

3+
use crate::expressions::tree::models::Expression;
34
use crate::guard::models::ComplexityGuardOptions;
45

56
pub(crate) struct GuardJsonTestCase {
@@ -57,3 +58,11 @@ pub(crate) struct TreeOperationTestCase {
5758
pub(crate) operation: fn(Vec<crate::expressions::tree::models::Expression>) -> bool,
5859
pub(crate) expected_completed: bool,
5960
}
61+
62+
pub(crate) struct ChainStepTestCase {
63+
pub(crate) description: &'static str,
64+
/// Wraps the chain so far in one more node of the kind under test.
65+
pub(crate) step: fn(Expression) -> Expression,
66+
/// The chain's depth after 100,000 steps, which the drop must survive.
67+
pub(crate) expected_depth: usize,
68+
}

0 commit comments

Comments
 (0)