Skip to content

Commit 4505bbb

Browse files
authored
perf: remove redundant graph-phase scans and parallelise source and resource SQL preparation (#1137)
* perf: index graph-phase lookups that scanned every model * perf: skip redundant set-operation, splice and shape-digest work * perf: parse source expressions and prepare resource SQL in parallel * perf: create the resource SQL pool only when needed and keep error order under panics
1 parent f100db2 commit 4505bbb

15 files changed

Lines changed: 397 additions & 99 deletions

File tree

‎crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! Python dict semantics over ordered pairs, and the topological analysis waves.
22
33
use std::collections::{BTreeSet, HashMap};
4+
use std::sync::OnceLock;
45

56
use sha2::{Digest, Sha256};
67

@@ -80,7 +81,7 @@ pub(crate) struct ShapeTable {
8081
/// Each name at its first insertion, Python's dict iteration order.
8182
order: Vec<String>,
8283
/// Digest chained over every insertion in order, so equal chains mean equal tables.
83-
chain: [u8; 32],
84+
chain: OnceLock<[u8; 32]>,
8485
}
8586

8687
impl ShapeTable {
@@ -92,13 +93,12 @@ impl ShapeTable {
9293
None => table.insert(name, shape.clone()),
9394
}
9495
}
95-
table.chain = shapes_chain(&table.ordered());
9696
table
9797
}
9898

9999
/// The digest of every shape in order, which changes whenever the table does.
100100
pub(crate) fn chain(&self) -> [u8; 32] {
101-
self.chain
101+
*self.chain.get_or_init(|| shapes_chain(&self.ordered()))
102102
}
103103

104104
fn insert(&mut self, name: &str, shape: Pairs) {
@@ -126,7 +126,9 @@ impl ShapeTable {
126126
/// Python's `setdefault`.
127127
pub(crate) fn set_default(&mut self, name: &str, shape: Pairs) {
128128
if !self.shapes.contains_key(name) {
129-
self.chain = chained(&self.chain, name, &shape);
129+
if let Some(chain) = self.chain.get() {
130+
self.chain = OnceLock::from(chained(chain, name, &shape));
131+
}
130132
self.insert(name, shape);
131133
}
132134
}
Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,18 @@
11
//! Python's `re.search(r"\b(?:UNION|INTERSECT|EXCEPT)\b", sql, re.IGNORECASE) is not None`.
22
3-
use sqlbuild_core::text::main::ignorecase_key::ignorecase_key;
43
use sqlbuild_core::text::main::is_word_char::is_word_char;
54

65
use crate::assembly::analysis_session::constants::SET_OPERATION_KEYWORDS;
76

8-
/// Whether `sql` names a set-operation keyword as a whole word, in any case.
7+
/// Whether a whole word of `sql` is a set-operation keyword; only ASCII letters fold to its letters.
98
pub fn names_set_operation(sql: &str) -> bool {
10-
let characters: Vec<char> = sql.chars().collect();
11-
for start in 0..characters.len() {
12-
if start > 0 && is_word_char(characters[start - 1]) {
13-
continue;
14-
}
15-
for keyword in SET_OPERATION_KEYWORDS {
16-
let end = start + keyword.len();
17-
if end > characters.len() {
18-
continue;
19-
}
20-
let matches = characters[start..end]
21-
.iter()
22-
.zip(keyword.chars())
23-
.all(|(character, letter)| ignorecase_key(*character) == letter);
24-
let bounded = characters.get(end).is_none_or(|next| !is_word_char(*next));
25-
if matches && bounded {
26-
return true;
27-
}
28-
}
29-
}
30-
false
9+
sql.split(|character: char| !is_word_char(character))
10+
.any(is_set_operation_keyword)
11+
}
12+
13+
/// Whether `word` is a set-operation keyword, ignoring ASCII case.
14+
fn is_set_operation_keyword(word: &str) -> bool {
15+
SET_OPERATION_KEYWORDS
16+
.iter()
17+
.any(|keyword| word.eq_ignore_ascii_case(keyword))
3118
}

‎crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ mod helpers;
1212
mod mappings;
1313
#[path = "tests/test_model_columns.rs"]
1414
mod model_columns;
15+
#[path = "tests/test_names_set_operation.rs"]
16+
mod names_set_operation;
1517
#[path = "tests/test_native_dialect.rs"]
1618
mod native_dialect;
1719
#[path = "tests/test_reference_facts.rs"]
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
use sqlbuild_core::text::main::ignorecase_key::ignorecase_key;
2+
3+
use crate::assembly::analysis_session::main::names_set_operation::names_set_operation;
4+
use crate::assembly::analysis_session::tests::test_types::{
5+
KeywordFoldingTestCase, NamesSetOperationTestCase,
6+
};
7+
8+
#[test]
9+
fn given_sql_when_checking_for_set_operations_then_matches_whole_words_in_any_case() {
10+
let test_cases = [
11+
NamesSetOperationTestCase {
12+
description: "a lowercase union is found",
13+
sql: "select 1 union all select 2",
14+
expected_set_operation: true,
15+
},
16+
NamesSetOperationTestCase {
17+
description: "a mixed-case intersect is found",
18+
sql: "SELECT a FROM t InTeRsEcT SELECT a FROM u",
19+
expected_set_operation: true,
20+
},
21+
NamesSetOperationTestCase {
22+
description: "a keyword at the end of the text is found",
23+
sql: "select a from t\nexcept",
24+
expected_set_operation: true,
25+
},
26+
NamesSetOperationTestCase {
27+
description: "a keyword inside an identifier is ignored",
28+
sql: "select reunion_id, union_count, except_flag from t",
29+
expected_set_operation: false,
30+
},
31+
NamesSetOperationTestCase {
32+
description: "a keyword followed by a non-ASCII letter is ignored",
33+
sql: "select 1 as unioné",
34+
expected_set_operation: false,
35+
},
36+
NamesSetOperationTestCase {
37+
description: "a keyword after a non-ASCII letter is ignored",
38+
sql: "select 1 as éunion",
39+
expected_set_operation: false,
40+
},
41+
NamesSetOperationTestCase {
42+
description: "a keyword after non-ASCII punctuation is found",
43+
sql: "select 1 «union» select 2",
44+
expected_set_operation: true,
45+
},
46+
NamesSetOperationTestCase {
47+
description: "text without a keyword is not a set operation",
48+
sql: "select ü, ß from t",
49+
expected_set_operation: false,
50+
},
51+
];
52+
53+
for test_case in test_cases {
54+
assert_eq!(
55+
names_set_operation(test_case.sql),
56+
test_case.expected_set_operation,
57+
"{}",
58+
test_case.description
59+
);
60+
}
61+
}
62+
63+
#[test]
64+
fn given_non_ascii_characters_when_folded_then_none_matches_a_keyword_letter() {
65+
let test_cases = [
66+
KeywordFoldingTestCase {
67+
description: "no non-ASCII character folds to a letter of 'union'",
68+
keyword: "union",
69+
expected_non_ascii_folds: Vec::new(),
70+
},
71+
KeywordFoldingTestCase {
72+
description: "no non-ASCII character folds to a letter of 'intersect'",
73+
keyword: "intersect",
74+
expected_non_ascii_folds: Vec::new(),
75+
},
76+
KeywordFoldingTestCase {
77+
description: "no non-ASCII character folds to a letter of 'except'",
78+
keyword: "except",
79+
expected_non_ascii_folds: Vec::new(),
80+
},
81+
];
82+
83+
for test_case in test_cases {
84+
let folds: Vec<char> = (0x80..=0x10FFFF_u32)
85+
.filter_map(char::from_u32)
86+
.filter(|character| test_case.keyword.contains(ignorecase_key(*character)))
87+
.collect();
88+
89+
assert_eq!(
90+
folds, test_case.expected_non_ascii_folds,
91+
"{}",
92+
test_case.description
93+
);
94+
}
95+
}

‎crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,18 @@ pub(crate) struct ReferenceFactsTestCase {
178178
pub(crate) expected_required: &'static [&'static str],
179179
}
180180

181+
pub(crate) struct NamesSetOperationTestCase {
182+
pub(crate) description: &'static str,
183+
pub(crate) sql: &'static str,
184+
pub(crate) expected_set_operation: bool,
185+
}
186+
187+
pub(crate) struct KeywordFoldingTestCase {
188+
pub(crate) description: &'static str,
189+
pub(crate) keyword: &'static str,
190+
pub(crate) expected_non_ascii_folds: Vec<char>,
191+
}
192+
181193
pub(crate) struct NativeDialectTestCase {
182194
pub(crate) description: &'static str,
183195
pub(crate) dialect: Option<&'static str>,

‎crates/sqlbuild-analysis/src/compiled_project/_helpers/splice.rs‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,29 @@
33
use crate::compiled_project::models::{DeclarationSplice, SplicedSpan, SplicedSql};
44

55
pub(crate) fn splice_declarations(sql: &str, splices: &[DeclarationSplice]) -> SplicedSql {
6+
if sql.is_ascii() {
7+
return spliced(sql.len(), splices, |output, start, end| {
8+
output.push_str(&sql[start..end]);
9+
});
10+
}
611
let characters: Vec<char> = sql.chars().collect();
7-
let mut output: String = String::with_capacity(sql.len());
12+
spliced(characters.len(), splices, |output, start, end| {
13+
output.extend(&characters[start..end]);
14+
})
15+
}
16+
17+
/// The splice over text of `length` characters, `copy` appending each kept character range.
18+
fn spliced(
19+
length: usize,
20+
splices: &[DeclarationSplice],
21+
copy: impl Fn(&mut String, usize, usize),
22+
) -> SplicedSql {
23+
let mut output: String = String::with_capacity(length);
824
let mut spans: Vec<SplicedSpan> = Vec::with_capacity(splices.len());
925
let mut cursor: usize = 0;
1026
let mut output_length: usize = 0;
1127
for splice in splices {
12-
output.extend(&characters[cursor..splice.start]);
28+
copy(&mut output, cursor, splice.start);
1329
output_length += splice.start - cursor;
1430
let replacement_length: usize = splice.replacement.chars().count();
1531
output.push_str(&splice.replacement);
@@ -22,7 +38,7 @@ pub(crate) fn splice_declarations(sql: &str, splices: &[DeclarationSplice]) -> S
2238
output_length += replacement_length;
2339
cursor = splice.end;
2440
}
25-
output.extend(&characters[cursor..]);
41+
copy(&mut output, cursor, length);
2642
SplicedSql { sql: output, spans }
2743
}
2844

‎crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
33
use std::collections::{BTreeSet, HashMap, HashSet};
44

5+
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
56
use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics;
7+
use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic;
68

79
use crate::semantic_checks::_helpers::explanation::opt_outs::unneeded_opt_out;
810
use crate::semantic_checks::_helpers::resource_sql::help::{HelpContext, resource_sql_help};
@@ -204,16 +206,31 @@ pub(crate) fn resource_sql_findings(
204206
request: &ResourceSqlRequest,
205207
catalog: &ProjectCatalog,
206208
) -> Result<Vec<ResourceSqlDiagnostic>, ResourceSqlFailure> {
207-
let checked = resources(request);
209+
let pool = || catalog.analysis_pool();
210+
let checked = resources(request, &pool).map_err(ResourceSqlFailure::Internal)?;
208211
if checked.is_empty() {
209212
return Ok(Vec::new());
210213
}
211214
let dialect = request.dialect.as_deref();
212-
let mut normalized: HashMap<&str, String> = HashMap::new();
213-
for resource in &checked {
214-
if !normalized.contains_key(resource.sql.as_str()) {
215-
normalized.insert(&resource.sql, cleaned_sql(&resource.sql, dialect)?);
216-
}
215+
let mut seen: HashSet<&str> = HashSet::with_capacity(checked.len());
216+
let unique: Vec<&str> = checked
217+
.iter()
218+
.map(|resource| resource.sql.as_str())
219+
.filter(|sql| seen.insert(sql))
220+
.collect();
221+
let cleaned: Vec<Result<String, ResourceSqlFailure>> =
222+
pool().map_err(ResourceSqlFailure::Internal)?.install(|| {
223+
unique
224+
.par_iter()
225+
.map(|sql| {
226+
catch_compiler_panic(|| Ok(cleaned_sql(sql, dialect)))
227+
.unwrap_or_else(|panic| Err(ResourceSqlFailure::Internal(panic)))
228+
})
229+
.collect()
230+
});
231+
let mut normalized: HashMap<&str, String> = HashMap::with_capacity(unique.len());
232+
for (sql, cleaned) in unique.into_iter().zip(cleaned) {
233+
normalized.insert(sql, cleaned?);
217234
}
218235
let shapes: HashMap<&str, &[(String, String)]> = request
219236
.shapes

0 commit comments

Comments
 (0)