Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use sqlquack_semantics::lineage::models::{
LineageContext, NameFinding, Nullability, OutputLineage, QueryLineage, SourceColumn,
TransformKind,
};
use sqlquack_semantics::{statement_lineage, statement_name_findings};
use sqlquack_semantics::statement_lineage_and_findings;
use sqlquack_syntax::Expression;

use crate::analysis_route::constants::{
Expand Down Expand Up @@ -37,7 +37,8 @@ pub(crate) fn native_analysis<D: NativeDialect + ?Sized>(
schema: input.schema,
sql: Some(input.sql),
};
let lineage: QueryLineage = statement_lineage(statement, context);
let (lineage, findings): (QueryLineage, Vec<NameFinding>) =
statement_lineage_and_findings(statement, context);
NativeAnalysis {
columns: lineage
.outputs
Expand All @@ -56,11 +57,7 @@ pub(crate) fn native_analysis<D: NativeDialect + ?Sized>(
.collect(),
has_star: lineage.projects_star,
star_resolved: !lineage.open_star,
diagnostics: diagnostics(
dialect.name(),
input,
statement_name_findings(statement, context),
),
diagnostics: diagnostics(dialect.name(), input, findings),
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
//! One native analysis shared by the models of a wave whose analyses differ only in names.

use std::collections::{HashMap, HashSet};
use std::hash::{Hash, Hasher};

use rayon::ThreadPool;
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
Expand Down Expand Up @@ -436,11 +437,23 @@ fn share_key(
name_order(&names, |name| name.to_owned()),
name_order(&names, str::to_ascii_uppercase),
];
let mut hasher: Sha256 = Sha256::new();
hasher.update(sql.len().to_le_bytes());
hasher.update(sql.as_bytes());
hasher.update(format!("{:?}", (shaped, names.len(), name_orders)).as_bytes());
Some(hasher.finalize().into())
let mut hasher: DigestHasher = DigestHasher(Sha256::new());
(sql, shaped, names.len(), name_orders).hash(&mut hasher);
Some(hasher.0.finalize().into())
}

/// Feeds `Hash` output into a SHA-256 digest, so a key hashes its fields without formatting them.
struct DigestHasher(Sha256);

impl Hasher for DigestHasher {
fn write(&mut self, bytes: &[u8]) {
self.0.update(bytes);
}

/// Unused: the key is the digest, read with `finalize`.
fn finish(&self) -> u64 {
0
}
}

/// The positions of `names` sorted by each name as `spelled`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,26 +318,24 @@ fn typed_columns(
columns: &Pairs,
known: Option<&Pairs>,
) -> Vec<SchemaColumn> {
let mut spelled: HashMap<&str, &str> = HashMap::new();
for (name, value) in known.into_iter().flatten() {
let _ = spelled.entry(name.as_str()).or_insert(value.as_str());
}
columns
.iter()
.map(|(name, data_type)| {
schema_column(
name,
(!data_type.is_empty() && data_type != UNKNOWN_TYPE)
.then(|| dialect.session_type(data_type)),
column_nullability(known, name),
column_nullability(spelled.get(name.as_str()).copied()),
)
})
.collect()
}

fn column_nullability(known: Option<&Pairs>, column: &str) -> Nullability {
let spelled: Option<&str> = known.and_then(|pairs| {
pairs
.iter()
.find(|(name, _)| name == column)
.map(|(_, value)| value.as_str())
});
fn column_nullability(spelled: Option<&str>) -> Nullability {
match spelled {
Some("non_null") => Nullability::NonNull,
Some("nullable") => Nullability::Nullable,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,9 @@ impl AnalysisSession {
)
})
.collect();
let pool: Arc<ThreadPool> = self.catalog.native.analysis_pool()?;
let findings: Vec<Vec<SqlMacroFinding>> =
check_sql_macro_arguments(&self.request.dialect, &batch);
check_sql_macro_arguments(&self.request.dialect, &batch, &pool);
for ((model, _, _), found) in inputs.iter().zip(findings) {
if let Some(outcome) = self.outcomes[*model].as_mut() {
outcome.sql_macro_findings = found;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,21 @@ impl MappedSql {
}
}

fn slice(&self, start: usize, end: usize) -> Self {
let first = self.sql[..start].chars().count();
let last = first + self.sql[start..end].chars().count();
/// The text from byte `start` to `end`, with `cursor` counting characters along `self.sql`.
fn slice(&self, cursor: &mut CharCursor, start: usize, end: usize) -> Self {
let first: usize = cursor.at(&self.sql, start);
let last: usize = cursor.at(&self.sql, end);
Self {
sql: self.sql[start..end].to_owned(),
offsets: self.offsets[first..=last].to_vec(),
}
}

/// The source offset of the character at byte `byte`.
fn anchor(&self, cursor: &mut CharCursor, byte: usize) -> usize {
self.offsets[cursor.at(&self.sql, byte)]
}

fn generated(text: String, anchor: usize) -> Self {
Self {
offsets: vec![anchor; text.chars().count() + 1],
Expand All @@ -81,18 +87,35 @@ impl MappedSql {
if edits.is_empty() {
return self;
}
let mut result = self.slice(0, 0);
let mut cursor: CharCursor = CharCursor::default();
let mut result = self.slice(&mut cursor, 0, 0);
let mut previous = 0;
for (start, end, replacement) in edits {
result.append(self.slice(previous, start));
result.append(self.slice(&mut cursor, previous, start));
result.append(replacement);
previous = end;
}
result.append(self.slice(previous, self.sql.len()));
result.append(self.slice(&mut cursor, previous, self.sql.len()));
result
}
}

/// The character index of byte offsets read in increasing order, counted on from the last one read.
#[derive(Default)]
struct CharCursor {
byte: usize,
chars: usize,
}

impl CharCursor {
/// The number of characters of `sql` before byte `byte`, which is at or after the last one read.
fn at(&mut self, sql: &str, byte: usize) -> usize {
self.chars += sql[self.byte..byte].chars().count();
self.byte = byte;
self.chars
}
}

pub(super) fn normalize(
sql: &str,
dialect: &str,
Expand All @@ -101,6 +124,7 @@ pub(super) fn normalize(
) -> Result<MappedSql, String> {
let mut result = MappedSql::new(sql);
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
for captures in REFERENCES
.as_ref()
.map_err(Clone::clone)?
Expand All @@ -111,15 +135,16 @@ pub(super) fn normalize(
let replacement = if &captures[1] != DBT_REF && stubs.contains_key(name.as_str()) {
MappedSql::generated(
stubs[name.as_str()].clone(),
result.offsets[result.sql[..name.start()].chars().count()],
result.anchor(&mut cursor, name.start()),
)
} else {
result.slice(name.start(), name.end())
result.slice(&mut cursor, name.start(), name.end())
};
edits.push((whole.start(), whole.end(), replacement));
}
result = result.edits(edits);
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
let mut consumed_until = 0;
let non_code: Vec<(usize, usize)> =
dialect_non_code_ranges(&result.sql, &dialect_lexical_syntax(dialect));
Expand Down Expand Up @@ -157,7 +182,7 @@ pub(super) fn normalize(
edits.push((
start,
end,
MappedSql::generated(stub, result.offsets[result.sql[..start].chars().count()]),
MappedSql::generated(stub, result.anchor(&mut cursor, start)),
));
consumed_until = end;
}
Expand All @@ -166,6 +191,7 @@ pub(super) fn normalize(
result = normalize_variant(result)?;
}
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
for captures in PLACEHOLDERS
.as_ref()
.map_err(Clone::clone)?
Expand All @@ -176,10 +202,7 @@ pub(super) fn normalize(
edits.push((
whole.start(),
whole.end(),
MappedSql::generated(
value.clone(),
result.offsets[result.sql[..whole.start()].chars().count()],
),
MappedSql::generated(value.clone(), result.anchor(&mut cursor, whole.start())),
));
}
}
Expand Down Expand Up @@ -213,10 +236,11 @@ pub(crate) fn normalize_native_sql_mapped(
placeholders: &[(String, String)],
) -> Result<(String, Vec<usize>), String> {
let native: MappedSql = native_mapped(sql, dialect, stub_prefix, &HashMap::new())?;
let mut cursor: CharCursor = CharCursor::default();
let edits: Vec<Edit> = placeholder_default_edits(&native.sql, placeholders)
.into_iter()
.map(|(start, end, default)| {
let anchor: usize = native.offsets[native.sql[..start].chars().count()];
let anchor: usize = native.anchor(&mut cursor, start);
(start, end, MappedSql::generated(default.to_owned(), anchor))
})
.collect();
Expand All @@ -237,6 +261,7 @@ fn native_mapped(
let non_code: Vec<(usize, usize)> =
dialect_non_code_ranges(&normalized.sql, &dialect_lexical_syntax(dialect));
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
for captures in stubs.captures_iter(&normalized.sql) {
let whole = captures.get(0).ok_or("missing stub match")?;
let quoted: bool = non_code
Expand All @@ -255,7 +280,7 @@ fn native_mapped(
(None, Some(parameter)) => format!("'{HOOK_PARAMETER_PREFIX}{}'", parameter.as_str()),
(None, None) => continue,
};
let anchor: usize = normalized.offsets[normalized.sql[..whole.start()].chars().count()];
let anchor: usize = normalized.anchor(&mut cursor, whole.start());
edits.push((
whole.start(),
whole.end(),
Expand Down Expand Up @@ -337,6 +362,8 @@ fn matching_bracket(sql: &str, start: usize) -> Result<Option<usize>, String> {
fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
let sql = &input.sql;
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
let mut key_cursor: CharCursor = CharCursor::default();
let mut index = 0;
while index < sql.len() {
if let Some(end) = skip_non_code(sql, index)? {
Expand All @@ -359,7 +386,7 @@ fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
index = close + 1;
continue;
}
let key = normalize_variant(input.slice(index + 1, close))?;
let key = normalize_variant(input.slice(&mut key_cursor, index + 1, close))?;
let mut base_end = index;
loop {
base_end = sql[..base_end].trim_end().len();
Expand All @@ -376,17 +403,17 @@ fn normalize_variant(input: MappedSql) -> Result<MappedSql, String> {
.map_err(Clone::clone)?
.find(&sql[..base_end])
{
let anchor = input.offsets[sql[..base.start()].chars().count()];
let anchor = input.anchor(&mut cursor, base.start());
let mut replacement = MappedSql::generated("GET(".to_owned(), anchor);
replacement.append(input.slice(base.start(), index));
replacement.append(input.slice(&mut cursor, base.start(), index));
replacement.append(MappedSql::generated(
", ".to_owned(),
input.offsets[sql[..index].chars().count()],
input.anchor(&mut cursor, index),
));
replacement.append(key);
replacement.append(MappedSql::generated(
")".to_owned(),
input.offsets[sql[..close].chars().count()],
input.anchor(&mut cursor, close),
));
edits.push((base.start(), close + 1, replacement));
} else if key.sql != sql[index + 1..close] {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use crate::semantic_validation::_helpers::normalization::normalize_native_sql_mapped;
use crate::semantic_validation::models::BindingPositions;
use crate::semantic_validation::tests::test_types::{
UniqueWordOffsetTestCase, UniqueWordScalingTestCase,
MappedNormalizationTestCase, UniqueWordOffsetTestCase, UniqueWordScalingTestCase,
};
use std::sync::OnceLock;
use std::time::{Duration, Instant};
Expand Down Expand Up @@ -109,3 +110,52 @@ fn given_many_unique_identifiers_when_locating_each_then_work_stays_linear() {
);
}
}

#[test]
fn given_non_ascii_text_around_several_edits_when_normalizing_then_copied_characters_map_to_their_source()
-> Result<(), String> {
let test_cases = [
MappedNormalizationTestCase {
description: "references between strings in other scripts",
dialect: "duckdb",
sql: "SELECT '\u{e9}' AS a, '\u{20ac}' AS b FROM __ref(\"orders\") AS o, \
__ref(\"customers\") AS c WHERE o.note = '\u{fc}' AND c.n = '\u{1f600}'",
expected_fragment: "FROM orders AS o",
},
MappedNormalizationTestCase {
description: "Snowflake variant paths with computed keys holding other scripts",
dialect: "snowflake",
sql: "SELECT '\u{e9}' AS a, v:a['x' || '\u{20ac}'] AS b, w:b['\u{df}' || 'k'] AS c \
FROM __ref(\"orders\") AS o WHERE o.n = '\u{fc}' || '\u{1f600}'",
expected_fragment: "GET(",
},
];
for test_case in test_cases {
let source: Vec<char> = test_case.sql.chars().collect();
let (normalized, offsets) =
normalize_native_sql_mapped(test_case.sql, test_case.dialect, "__stub_", &[])?;
let output: Vec<char> = normalized.chars().collect();
assert!(
normalized.contains(test_case.expected_fragment),
"{}: {normalized}",
test_case.description
);

assert_eq!(offsets.len(), output.len() + 1, "{}", test_case.description);
let copied: Vec<(char, Option<char>)> = output
.iter()
.zip(&offsets)
.filter(|(character, _)| !character.is_ascii())
.map(|(character, offset)| (*character, source.get(*offset).copied()))
.collect();
let authored: usize = source
.iter()
.filter(|character| !character.is_ascii())
.count();
assert_eq!(copied.len(), authored, "{}", test_case.description);
for (character, mapped) in copied {
assert_eq!(Some(character), mapped, "{}", test_case.description);
}
}
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,10 @@ pub(crate) struct TemporalCastDialectTestCase {
pub(crate) dialect: polyglot_sql::DialectType,
pub(crate) expected_warnings: usize,
}

pub(crate) struct MappedNormalizationTestCase {
pub(crate) description: &'static str,
pub(crate) dialect: &'static str,
pub(crate) sql: &'static str,
pub(crate) expected_fragment: &'static str,
}
Loading
Loading