Skip to content

Commit e29ac02

Browse files
committed
perf(analysis): check SQL-macro arguments on the analysis pool and drop duplicate work
The probes that check each typed SQL-macro call ran one model at a time on the session thread while the analysis pool sat idle; each is a parse and a lineage analysis, so in projects with many typed calls they were about half of all analysis time. Each model's probes now run on the pool, and a probe's relation lookup reads a map built once per schema instead of lower-casing, formatting and scanning every relation. Duplicate work inside the per-model analysis is gone too: - lineage and name findings come from one SQLQuack analysis instead of two; - the share key feeds the hasher its fields instead of a Debug string of every upstream column; - normalisation counts characters forward from the last edit instead of from the start of the SQL for every match, which was quadratic in long models; - column nullability is read from a map instead of a scan per column. Results are unchanged: findings, compile JSON and compiled files are byte-identical before and after.
1 parent f98235d commit e29ac02

14 files changed

Lines changed: 303 additions & 85 deletions

File tree

‎crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_analysis.rs‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use sqlquack_semantics::lineage::models::{
55
LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, SourceColumn,
66
TransformKind,
77
};
8-
use sqlquack_semantics::{statement_lineage, statement_name_findings};
8+
use sqlquack_semantics::statement_lineage_and_findings;
99
use sqlquack_syntax::Expression;
1010

1111
use crate::analysis_route::constants::{
@@ -37,7 +37,8 @@ pub(crate) fn native_analysis<D: NativeDialect + ?Sized>(
3737
schema: input.schema,
3838
sql: Some(input.sql),
3939
};
40-
let lineage: QueryLineage = statement_lineage(statement, context);
40+
let (lineage, findings): (QueryLineage, Vec<NameFinding>) =
41+
statement_lineage_and_findings(statement, context);
4142
NativeAnalysis {
4243
columns: lineage
4344
.outputs
@@ -56,11 +57,7 @@ pub(crate) fn native_analysis<D: NativeDialect + ?Sized>(
5657
.collect(),
5758
has_star: lineage.projects_star,
5859
star_resolved: !lineage.open_star,
59-
diagnostics: diagnostics(
60-
dialect.name(),
61-
input,
62-
statement_name_findings(statement, context),
63-
),
60+
diagnostics: diagnostics(dialect.name(), input, findings),
6461
}
6562
}
6663

‎crates/sqlbuild-analysis/src/analysis_route/_helpers/analysis/native_sharing.rs‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
//! One native analysis shared by the models of a wave whose analyses differ only in names.
22
33
use std::collections::{HashMap, HashSet};
4+
use std::hash::{Hash, Hasher};
45

56
use rayon::ThreadPool;
67
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
@@ -436,11 +437,23 @@ fn share_key(
436437
name_order(&names, |name| name.to_owned()),
437438
name_order(&names, str::to_ascii_uppercase),
438439
];
439-
let mut hasher: Sha256 = Sha256::new();
440-
hasher.update(sql.len().to_le_bytes());
441-
hasher.update(sql.as_bytes());
442-
hasher.update(format!("{:?}", (shaped, names.len(), name_orders)).as_bytes());
443-
Some(hasher.finalize().into())
440+
let mut hasher: DigestHasher = DigestHasher(Sha256::new());
441+
(sql, shaped, names.len(), name_orders).hash(&mut hasher);
442+
Some(hasher.0.finalize().into())
443+
}
444+
445+
/// Feeds `Hash` output into a SHA-256 digest, so a key hashes its fields without formatting them.
446+
struct DigestHasher(Sha256);
447+
448+
impl Hasher for DigestHasher {
449+
fn write(&mut self, bytes: &[u8]) {
450+
self.0.update(bytes);
451+
}
452+
453+
/// Unused: the key is the digest, read with `finalize`.
454+
fn finish(&self) -> u64 {
455+
0
456+
}
444457
}
445458

446459
/// The positions of `names` sorted by each name as `spelled`.

‎crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/mappings.rs‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -318,26 +318,24 @@ fn typed_columns(
318318
columns: &Pairs,
319319
known: Option<&Pairs>,
320320
) -> Vec<SchemaColumn> {
321+
let mut spelled: HashMap<&str, &str> = HashMap::new();
322+
for (name, value) in known.into_iter().flatten() {
323+
let _ = spelled.entry(name.as_str()).or_insert(value.as_str());
324+
}
321325
columns
322326
.iter()
323327
.map(|(name, data_type)| {
324328
schema_column(
325329
name,
326330
(!data_type.is_empty() && data_type != UNKNOWN_TYPE)
327331
.then(|| dialect.session_type(data_type)),
328-
column_nullability(known, name),
332+
column_nullability(spelled.get(name.as_str()).copied()),
329333
)
330334
})
331335
.collect()
332336
}
333337

334-
fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability {
335-
let spelled: Option<&str> = known.and_then(|pairs| {
336-
pairs
337-
.iter()
338-
.find(|(name, _)| name == column)
339-
.map(|(_, value)| value.as_str())
340-
});
338+
fn column_nullability(spelled: Option<&str>) -> Nullability {
341339
match spelled {
342340
Some("non_null") => Nullability::NonNull,
343341
Some("nullable") => Nullability::Nullable,

‎crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,9 @@ impl AnalysisSession {
394394
)
395395
})
396396
.collect();
397+
let pool: Arc<ThreadPool> = self.catalog.native.analysis_pool()?;
397398
let findings: Vec<Vec<SqlMacroFinding>> =
398-
check_sql_macro_arguments(&self.request.dialect, &batch);
399+
check_sql_macro_arguments(&self.request.dialect, &batch, &pool);
399400
for ((model, _, _), found) in inputs.iter().zip(findings) {
400401
if let Some(outcome) = self.outcomes[*model].as_mut() {
401402
outcome.sql_macro_findings = found;

‎crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs‎

Lines changed: 47 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,21 @@ impl MappedSql {
5555
}
5656
}
5757

58-
fn slice(&self, start: usize, end: usize) -> Self {
59-
let first = self.sql[..start].chars().count();
60-
let last = first + self.sql[start..end].chars().count();
58+
/// The text from byte `start` to `end`, with `cursor` counting characters along `self.sql`.
59+
fn slice(&self, cursor: &mut CharCursor, start: usize, end: usize) -> Self {
60+
let first: usize = cursor.at(&self.sql, start);
61+
let last: usize = cursor.at(&self.sql, end);
6162
Self {
6263
sql: self.sql[start..end].to_owned(),
6364
offsets: self.offsets[first..=last].to_vec(),
6465
}
6566
}
6667

68+
/// The source offset of the character at byte `byte`.
69+
fn anchor(&self, cursor: &mut CharCursor, byte: usize) -> usize {
70+
self.offsets[cursor.at(&self.sql, byte)]
71+
}
72+
6773
fn generated(text: String, anchor: usize) -> Self {
6874
Self {
6975
offsets: vec![anchor; text.chars().count() + 1],
@@ -81,18 +87,35 @@ impl MappedSql {
8187
if edits.is_empty() {
8288
return self;
8389
}
84-
let mut result = self.slice(0, 0);
90+
let mut cursor: CharCursor = CharCursor::default();
91+
let mut result = self.slice(&mut cursor, 0, 0);
8592
let mut previous = 0;
8693
for (start, end, replacement) in edits {
87-
result.append(self.slice(previous, start));
94+
result.append(self.slice(&mut cursor, previous, start));
8895
result.append(replacement);
8996
previous = end;
9097
}
91-
result.append(self.slice(previous, self.sql.len()));
98+
result.append(self.slice(&mut cursor, previous, self.sql.len()));
9299
result
93100
}
94101
}
95102

103+
/// The character index of byte offsets read in increasing order, counted on from the last one read.
104+
#[derive(Default)]
105+
struct CharCursor {
106+
byte: usize,
107+
chars: usize,
108+
}
109+
110+
impl CharCursor {
111+
/// The number of characters of `sql` before byte `byte`, which is at or after the last one read.
112+
fn at(&mut self, sql: &str, byte: usize) -> usize {
113+
self.chars += sql[self.byte..byte].chars().count();
114+
self.byte = byte;
115+
self.chars
116+
}
117+
}
118+
96119
pub(super) fn normalize(
97120
sql: &str,
98121
dialect: &str,
@@ -101,6 +124,7 @@ pub(super) fn normalize(
101124
) -> Result<MappedSql, String> {
102125
let mut result = MappedSql::new(sql);
103126
let mut edits: Vec<Edit> = Vec::new();
127+
let mut cursor: CharCursor = CharCursor::default();
104128
for captures in REFERENCES
105129
.as_ref()
106130
.map_err(Clone::clone)?
@@ -111,15 +135,16 @@ pub(super) fn normalize(
111135
let replacement = if &captures[1] != DBT_REF && stubs.contains_key(name.as_str()) {
112136
MappedSql::generated(
113137
stubs[name.as_str()].clone(),
114-
result.offsets[result.sql[..name.start()].chars().count()],
138+
result.anchor(&mut cursor, name.start()),
115139
)
116140
} else {
117-
result.slice(name.start(), name.end())
141+
result.slice(&mut cursor, name.start(), name.end())
118142
};
119143
edits.push((whole.start(), whole.end(), replacement));
120144
}
121145
result = result.edits(edits);
122146
let mut edits: Vec<Edit> = Vec::new();
147+
let mut cursor: CharCursor = CharCursor::default();
123148
let mut consumed_until = 0;
124149
let non_code: Vec<(usize, usize)> =
125150
dialect_non_code_ranges(&result.sql, &dialect_lexical_syntax(dialect));
@@ -157,7 +182,7 @@ pub(super) fn normalize(
157182
edits.push((
158183
start,
159184
end,
160-
MappedSql::generated(stub, result.offsets[result.sql[..start].chars().count()]),
185+
MappedSql::generated(stub, result.anchor(&mut cursor, start)),
161186
));
162187
consumed_until = end;
163188
}
@@ -166,6 +191,7 @@ pub(super) fn normalize(
166191
result = normalize_variant(result)?;
167192
}
168193
let mut edits: Vec<Edit> = Vec::new();
194+
let mut cursor: CharCursor = CharCursor::default();
169195
for captures in PLACEHOLDERS
170196
.as_ref()
171197
.map_err(Clone::clone)?
@@ -176,10 +202,7 @@ pub(super) fn normalize(
176202
edits.push((
177203
whole.start(),
178204
whole.end(),
179-
MappedSql::generated(
180-
value.clone(),
181-
result.offsets[result.sql[..whole.start()].chars().count()],
182-
),
205+
MappedSql::generated(value.clone(), result.anchor(&mut cursor, whole.start())),
183206
));
184207
}
185208
}
@@ -213,10 +236,11 @@ pub(crate) fn normalize_native_sql_mapped(
213236
placeholders: &[(String, String)],
214237
) -> Result<(String, Vec<usize>), String> {
215238
let native: MappedSql = native_mapped(sql, dialect, stub_prefix, &HashMap::new())?;
239+
let mut cursor: CharCursor = CharCursor::default();
216240
let edits: Vec<Edit> = placeholder_default_edits(&native.sql, placeholders)
217241
.into_iter()
218242
.map(|(start, end, default)| {
219-
let anchor: usize = native.offsets[native.sql[..start].chars().count()];
243+
let anchor: usize = native.anchor(&mut cursor, start);
220244
(start, end, MappedSql::generated(default.to_owned(), anchor))
221245
})
222246
.collect();
@@ -237,6 +261,7 @@ fn native_mapped(
237261
let non_code: Vec<(usize, usize)> =
238262
dialect_non_code_ranges(&normalized.sql, &dialect_lexical_syntax(dialect));
239263
let mut edits: Vec<Edit> = Vec::new();
264+
let mut cursor: CharCursor = CharCursor::default();
240265
for captures in stubs.captures_iter(&normalized.sql) {
241266
let whole = captures.get(0).ok_or("missing stub match")?;
242267
let quoted: bool = non_code
@@ -255,7 +280,7 @@ fn native_mapped(
255280
(None, Some(parameter)) => format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()),
256281
(None, None) => continue,
257282
};
258-
let anchor: usize = normalized.offsets[normalized.sql[..whole.start()].chars().count()];
283+
let anchor: usize = normalized.anchor(&mut cursor, whole.start());
259284
edits.push((
260285
whole.start(),
261286
whole.end(),
@@ -337,6 +362,8 @@ fn matching_bracket(sql: &str, start: usize) -> Result<Option<usize>, String> {
337362
fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
338363
let sql = &input.sql;
339364
let mut edits: Vec<Edit> = Vec::new();
365+
let mut cursor: CharCursor = CharCursor::default();
366+
let mut key_cursor: CharCursor = CharCursor::default();
340367
let mut index = 0;
341368
while index < sql.len() {
342369
if let Some(end) = skip_non_code(sql, index)? {
@@ -359,7 +386,7 @@ fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
359386
index = close + 1;
360387
continue;
361388
}
362-
let key = normalize_variant(input.slice(index + 1, close))?;
389+
let key = normalize_variant(input.slice(&mut key_cursor, index + 1, close))?;
363390
let mut base_end = index;
364391
loop {
365392
base_end = sql[..base_end].trim_end().len();
@@ -376,17 +403,17 @@ fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
376403
.map_err(Clone::clone)?
377404
.find(&sql[..base_end])
378405
{
379-
let anchor = input.offsets[sql[..base.start()].chars().count()];
406+
let anchor = input.anchor(&mut cursor, base.start());
380407
let mut replacement = MappedSql::generated("GET(".to_owned(), anchor);
381-
replacement.append(input.slice(base.start(), index));
408+
replacement.append(input.slice(&mut cursor, base.start(), index));
382409
replacement.append(MappedSql::generated(
383410
", ".to_owned(),
384-
input.offsets[sql[..index].chars().count()],
411+
input.anchor(&mut cursor, index),
385412
));
386413
replacement.append(key);
387414
replacement.append(MappedSql::generated(
388415
")".to_owned(),
389-
input.offsets[sql[..close].chars().count()],
416+
input.anchor(&mut cursor, close),
390417
));
391418
edits.push((base.start(), close + 1, replacement));
392419
} else if key.sql != sql[index + 1..close] {

‎crates/sqlbuild-analysis/src/semantic_validation/tests/test_positions.rs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped;
12
use crate::semantic_validation::models::BindingPositions;
23
use crate::semantic_validation::tests::test_types::{
3-
UniqueWordOffsetTestCase, UniqueWordScalingTestCase,
4+
MappedNormalizationTestCase, UniqueWordOffsetTestCase, UniqueWordScalingTestCase,
45
};
56
use std::sync::OnceLock;
67
use std::time::{Duration, Instant};
@@ -109,3 +110,52 @@ fn given_many_unique_identifiers_when_locating_each_then_work_stays_linear() {
109110
);
110111
}
111112
}
113+
114+
#[test]
115+
fn given_non_ascii_text_around_several_edits_when_normalizing_then_copied_characters_map_to_their_source()
116+
-> Result<(), String> {
117+
let test_cases = [
118+
MappedNormalizationTestCase {
119+
description: "references between strings in other scripts",
120+
dialect: "duckdb",
121+
sql: "SELECT '\u{e9}' AS a, '\u{20ac}' AS b FROM __ref(\"orders\") AS o, \
122+
__ref(\"customers\") AS c WHERE o.note = '\u{fc}' AND c.n = '\u{1f600}'",
123+
expected_fragment: "FROM orders AS o",
124+
},
125+
MappedNormalizationTestCase {
126+
description: "Snowflake variant paths with computed keys holding other scripts",
127+
dialect: "snowflake",
128+
sql: "SELECT '\u{e9}' AS a, v:a['x' || '\u{20ac}'] AS b, w:b['\u{df}' || 'k'] AS c \
129+
FROM __ref(\"orders\") AS o WHERE o.n = '\u{fc}' || '\u{1f600}'",
130+
expected_fragment: "GET(",
131+
},
132+
];
133+
for test_case in test_cases {
134+
let source: Vec<char> = test_case.sql.chars().collect();
135+
let (normalized, offsets) =
136+
normalize_native_sql_mapped(test_case.sql, test_case.dialect, "__stub_", &[])?;
137+
let output: Vec<char> = normalized.chars().collect();
138+
assert!(
139+
normalized.contains(test_case.expected_fragment),
140+
"{}: {normalized}",
141+
test_case.description
142+
);
143+
144+
assert_eq!(offsets.len(), output.len() + 1, "{}", test_case.description);
145+
let copied: Vec<(char, Option<char>)> = output
146+
.iter()
147+
.zip(&offsets)
148+
.filter(|(character, _)| !character.is_ascii())
149+
.map(|(character, offset)| (*character, source.get(*offset).copied()))
150+
.collect();
151+
let authored: usize = source
152+
.iter()
153+
.filter(|character| !character.is_ascii())
154+
.count();
155+
assert_eq!(copied.len(), authored, "{}", test_case.description);
156+
for (character, mapped) in copied {
157+
assert_eq!(Some(character), mapped, "{}", test_case.description);
158+
}
159+
}
160+
Ok(())
161+
}

‎crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,10 @@ pub(crate) struct TemporalCastDialectTestCase {
9292
pub(crate) dialect: polyglot_sql::DialectType,
9393
pub(crate) expected_warnings: usize,
9494
}
95+
96+
pub(crate) struct MappedNormalizationTestCase {
97+
pub(crate) description: &'static str,
98+
pub(crate) dialect: &'static str,
99+
pub(crate) sql: &'static str,
100+
pub(crate) expected_fragment: &'static str,
101+
}

0 commit comments

Comments
 (0)