Skip to content

Commit 7cc9233

Browse files
committed
feat(native): extract and classify scenario CTEs natively behind the preview engine
1 parent acf988a commit 7cc9233

14 files changed

Lines changed: 381 additions & 3 deletions

File tree

‎crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ struct TestRequest {
5858
}
5959

6060
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
61-
struct Cte(String, String);
61+
pub(crate) struct Cte(pub(crate) String, pub(crate) String);
6262

6363
#[derive(Debug, PartialEq, Eq, Serialize)]
6464
#[serde(tag = "kind", rename_all = "snake_case")]
@@ -105,6 +105,12 @@ pub(crate) fn extract_batch_json(request_json: &str) -> Result<String, String> {
105105
}
106106

107107
fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
108+
extract_ctes_with_quoting(sql, file).map(|(ctes, _)| ctes)
109+
}
110+
111+
/// The top-level CTEs and whether any CTE name was a quoted identifier.
112+
pub(crate) fn extract_ctes_with_quoting(sql: &str, file: &str) -> Result<(Vec<Cte>, bool), String> {
113+
let mut quoted = false;
108114
let mut index = skip_ignorable(sql, 0)?;
109115
index = consume_keyword(sql, index, "WITH").ok_or_else(|| {
110116
format!("SQL test '{file}' must declare mock CTEs and one __expected__<model> CTE in a top-level WITH clause")
@@ -116,6 +122,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
116122
let mut ctes: Vec<Cte> = Vec::new();
117123
let mut seen: HashSet<String> = HashSet::new();
118124
loop {
125+
quoted |= matches!(byte_at(sql, index), Some(b'"' | b'`'));
119126
let (name, end) = read_identifier(sql, index)
120127
.ok_or_else(|| format!("SQL test '{file}' expected a CTE name"))?;
121128
if !seen.insert(name.clone()) {
@@ -145,7 +152,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
145152
"SQL test '{file}' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them"
146153
));
147154
}
148-
Ok(ctes)
155+
Ok((ctes, quoted))
149156
}
150157

151158
fn classify(ctes: Vec<Cte>, file: &str, mode: &str) -> Result<Classified, String> {
@@ -568,7 +575,7 @@ fn implicit_projection_alias(expression: &str) -> Option<String> {
568575
(end == expression.len()).then_some(alias)
569576
}
570577

571-
fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> {
578+
pub(crate) fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> {
572579
let names: HashMap<String, String> = ctes
573580
.iter()
574581
.map(|cte| (cte.0.to_lowercase(), cte.0.clone()))

‎crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ pub mod markers;
99
pub mod planning;
1010
pub mod relation_markers;
1111
pub mod rendering;
12+
pub mod scenario_extraction;
Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
//! Python's `extract_sql_scenario_ctes` for scenarios its top-level scanner reads exactly.
2+
3+
use serde::Serialize;
4+
5+
use crate::compiler::_helpers::sql_tests::extraction::{
6+
Cte, extract_ctes_with_quoting, validate_independence,
7+
};
8+
9+
/// Characters whose Python case mapping reaches ASCII, which Python's keyword match would accept.
10+
const CASE_MAPPED_TO_ASCII: [char; 20] = [
11+
'\u{df}', '\u{130}', '\u{131}', '\u{149}', '\u{17f}', '\u{1f0}', '\u{1e96}', '\u{1e97}',
12+
'\u{1e98}', '\u{1e99}', '\u{1e9a}', '\u{1e9e}', '\u{212a}', '\u{fb00}', '\u{fb01}', '\u{fb02}',
13+
'\u{fb03}', '\u{fb04}', '\u{fb05}', '\u{fb06}',
14+
];
15+
16+
#[derive(Debug, Default, Serialize)]
17+
#[serde(rename_all = "camelCase")]
18+
struct ClassifiedScenario {
19+
authored: Vec<Cte>,
20+
expected: Vec<Cte>,
21+
assertions: Vec<Cte>,
22+
source_fixtures: Vec<String>,
23+
ref_fixtures: Vec<String>,
24+
seed_fixtures: Vec<String>,
25+
dbt_ref_fixtures: Vec<String>,
26+
expected_models: Vec<String>,
27+
assertion_names: Vec<String>,
28+
}
29+
30+
pub(crate) fn extract_scenario_json(sql: &str, file: &str) -> Result<Option<String>, String> {
31+
if sql.chars().any(|character| {
32+
matches!(character, '\u{1c}'..='\u{1f}') || CASE_MAPPED_TO_ASCII.contains(&character)
33+
}) {
34+
return Ok(None);
35+
}
36+
let Ok((ctes, quoted)) = extract_ctes_with_quoting(sql, file) else {
37+
return Ok(None);
38+
};
39+
if quoted || !ctes.iter().all(|cte| is_plain_identifier(&cte.0)) {
40+
return Ok(None);
41+
}
42+
if validate_independence(&ctes, file).is_err() {
43+
return Ok(None);
44+
}
45+
let Some(scenario) = classify(ctes) else {
46+
return Ok(None);
47+
};
48+
serde_json::to_string(&scenario)
49+
.map(Some)
50+
.map_err(|error| error.to_string())
51+
}
52+
53+
/// Python's `_read_identifier` reads only `[A-Za-z_][A-Za-z0-9_]*` the same way.
54+
fn is_plain_identifier(name: &str) -> bool {
55+
let mut bytes = name.bytes();
56+
bytes
57+
.next()
58+
.is_some_and(|first| first.is_ascii_alphabetic() || first == b'_')
59+
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
60+
}
61+
62+
fn classify(ctes: Vec<Cte>) -> Option<ClassifiedScenario> {
63+
let mut scenario: ClassifiedScenario = ClassifiedScenario::default();
64+
for cte in ctes {
65+
let name: String = cte.0.clone();
66+
if let Some(value) = name.strip_prefix("__source__") {
67+
scenario.source_fixtures.push(required(value)?);
68+
scenario.authored.push(cte);
69+
} else if let Some(value) = name.strip_prefix("__ref__") {
70+
scenario.ref_fixtures.push(required(value)?);
71+
scenario.authored.push(cte);
72+
} else if let Some(value) = name.strip_prefix("__seed__") {
73+
scenario.seed_fixtures.push(required(value)?);
74+
scenario.authored.push(cte);
75+
} else if let Some(value) = name.strip_prefix("__dbt_ref__") {
76+
scenario.dbt_ref_fixtures.push(required(value)?);
77+
scenario.authored.push(cte);
78+
} else if let Some(value) = name.strip_prefix("__expected__") {
79+
scenario.expected_models.push(required(value)?);
80+
scenario.expected.push(cte);
81+
} else if let Some(value) = name.strip_prefix("__assert__") {
82+
scenario.assertion_names.push(required(value)?);
83+
scenario.assertions.push(cte);
84+
} else if name.starts_with("__macro__") {
85+
return None;
86+
} else {
87+
scenario.authored.push(cte);
88+
}
89+
}
90+
let has_fixture: bool = !(scenario.source_fixtures.is_empty()
91+
&& scenario.ref_fixtures.is_empty()
92+
&& scenario.seed_fixtures.is_empty()
93+
&& scenario.dbt_ref_fixtures.is_empty());
94+
let has_check: bool =
95+
!(scenario.expected_models.is_empty() && scenario.assertion_names.is_empty());
96+
(has_fixture && has_check).then_some(scenario)
97+
}
98+
99+
fn required(value: &str) -> Option<String> {
100+
(!value.is_empty()).then(|| value.to_owned())
101+
}

‎crates/sqlbuild-analysis/src/compiler/main/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
pub mod sql_scenario_extraction;
12
pub mod sql_test_chain_resolution;
23
pub mod sql_test_difference_sampling;
34
pub mod sql_test_extraction;
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
use crate::compiler::_helpers;
2+
3+
/// Python's extracted scenario CTEs as JSON, or `None` where Python must extract the scenario.
4+
pub fn extract_scenario_json(sql: &str, file_label: &str) -> Result<Option<String>, String> {
5+
_helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label)
6+
}

‎crates/sqlbuild-analysis/src/compiler/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#[path = "tests/helpers.rs"]
22
mod helpers;
3+
#[path = "tests/test_sql_scenario_extraction.rs"]
4+
mod sql_scenario_extraction;
35
#[path = "tests/test_sql_test_cte_rename.rs"]
46
mod sql_test_cte_rename;
57
#[path = "tests/test_sql_test_cte_slices.rs"]
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
use crate::compiler::main::sql_scenario_extraction::extract_scenario_json;
2+
use crate::compiler::tests::test_types::ScenarioExtractionTestCase;
3+
4+
#[test]
5+
fn given_scenarios_when_extracting_natively_then_python_payloads_or_deferrals_are_returned() {
6+
let test_cases = [
7+
ScenarioExtractionTestCase {
8+
description: "fixtures, helpers, expected results and assertions are classified",
9+
sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), \
10+
__expected__orders AS (SELECT 1 AS id), __assert__positive AS (SELECT 1) SELECT 1",
11+
expected_json: Some(
12+
r#"{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}"#,
13+
),
14+
},
15+
ScenarioExtractionTestCase {
16+
description: "a macro mock defers so Python raises",
17+
sql: "WITH __source__raw AS (SELECT 1), __macro__tidy AS (SELECT 'x'), \
18+
__expected__orders AS (SELECT 1 AS id) SELECT 1",
19+
expected_json: None,
20+
},
21+
ScenarioExtractionTestCase {
22+
description: "a quoted CTE name defers to Python's scanner fallback",
23+
sql: "WITH \"__source__raw\" AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)",
24+
expected_json: None,
25+
},
26+
ScenarioExtractionTestCase {
27+
description: "a keyword Python would match by case mapping defers",
28+
sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)",
29+
expected_json: None,
30+
},
31+
ScenarioExtractionTestCase {
32+
description: "a scenario without fixtures defers so Python raises",
33+
sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1",
34+
expected_json: None,
35+
},
36+
];
37+
38+
for test_case in test_cases {
39+
assert_eq!(
40+
extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql"),
41+
Ok(test_case.expected_json.map(str::to_owned)),
42+
"{}",
43+
test_case.description
44+
);
45+
}
46+
}

‎crates/sqlbuild-analysis/src/compiler/tests/test_types.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,3 +118,9 @@ pub(crate) struct PlanShape<'a> {
118118
pub(crate) helper_name: &'a str,
119119
pub(crate) expected_sql: &'a str,
120120
}
121+
122+
pub(crate) struct ScenarioExtractionTestCase {
123+
pub(crate) description: &'static str,
124+
pub(crate) sql: &'static str,
125+
pub(crate) expected_json: Option<&'static str>,
126+
}

‎crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,23 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult<String
5050
.map_err(value_error)
5151
}
5252

53+
/// Python's extracted scenario CTEs as JSON, or `None` where Python must extract the scenario.
54+
#[pyfunction]
55+
fn extract_sql_scenario_json(
56+
py: Python<'_>,
57+
sql: &str,
58+
file_label: &str,
59+
) -> PyResult<Option<String>> {
60+
py.compiler_detach(|| {
61+
sqlbuild_analysis::compiler::main::sql_scenario_extraction::extract_scenario_json(
62+
sql, file_label,
63+
)
64+
})
65+
.map_err(value_error)
66+
}
67+
5368
pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
69+
module.add_function(wrap_pyfunction!(extract_sql_scenario_json, module)?)?;
5470
module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?;
5571
module.add_function(wrap_pyfunction!(plan_and_render_sql_tests_json, module)?)?;
5672
module.add_function(wrap_pyfunction!(resolve_sql_test_chains_json, module)?)?;

‎src/sqlbuild/_native.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ class SqlReferenceScanner:
175175
def scope_expected_model_names(
176176
sqls: list[str], syntax: dict[str, object]
177177
) -> list[list[str] | None]: ...
178+
def extract_sql_scenario_json(sql: str, file_label: str) -> str | None: ...
178179
def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> tuple[bool, int | None]: ...
179180
def scan_test_parameter_references(
180181
sql: str, declared: list[str]

0 commit comments

Comments
 (0)