Skip to content

Commit 1ee7c8f

Browse files
authored
feat!: compute query fingerprints from SQLQuack tokens (#1348)
Query fingerprints for generic, DuckDB, PostgreSQL, Snowflake and SQLite SQL now come from SQLQuack's tokenizers instead of Polyglot's: the lossless tokenizer for generic, DuckDB, Snowflake and SQLite, and the native PostgreSQL scanner for PostgreSQL. Reserved keywords and unquoted called names fold case outside name positions; quoted names, SQLBuild placeholders such as __ref, and CTE or INTO/TABLE column-list names keep their case. Parentheses are matched in one pass, so deep or unbalanced nesting stays linear. BigQuery, Databricks, SQL Server and ClickHouse keep the previous token stream until SQLQuack tokenizes them. BREAKING CHANGE: the first run after upgrading can plan some models as query_changed even though their SQL did not change. On DuckDB, Snowflake and SQLite this affects models that call an unquoted user-defined function, and on SQLite also models calling built-ins such as datetime, date or strftime in lower case; PostgreSQL models are unaffected. Plan that run: affected incremental and microbatch models whose replay_on_change is bounded or full replay or rebuild (even with full_refresh false), sqb build --changed reruns them, and models on the default forward policy only warn.
1 parent cebdfdc commit 1ee7c8f

17 files changed

Lines changed: 667 additions & 10 deletions

File tree

‎Cargo.lock‎

Lines changed: 33 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,11 @@ sqlbuild-recall = { path = "crates/sqlbuild-recall" }
174174
sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }
175175
sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
176176
sqlbuild-state = { path = "crates/sqlbuild-state" }
177-
# SQL lint Rules, quality Rules, the formatter and the canonical token stream.
177+
# SQL lint Rules, quality Rules and the formatter.
178178
sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" }
179+
# The lossless SQL tokenizer behind query fingerprints.
180+
sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
181+
sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" }
179182
stacker = "0.1"
180183
syn = { version = "2", features = ["full", "visit", "extra-traits"] }
181184
subtle = "2"

‎crates/sqlbuild-analysis/Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ sqlbuild-render.workspace = true
2626
sqlbuild-scopes.workspace = true
2727
sqlbuild-sqltext.workspace = true
2828
sqlclean.workspace = true
29+
sqlquack-syntax.workspace = true
30+
sqlquack-syntax-postgres.workspace = true
2931

3032
[lints]
3133
workspace = true
Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
//! The canonical token stream of SQL under SQLQuack's tokenizers.
2+
3+
use sqlclean::sql_tokens::main::is_unquoted_word::is_unquoted_word;
4+
use sqlclean::sql_tokens::main::name_positions::name_positions;
5+
use sqlquack_syntax::lossless::models::TokenKind;
6+
use sqlquack_syntax_postgres::scan::main::scan::scan;
7+
use sqlquack_syntax_postgres::scan::models::TokenKind as PostgresTokenKind;
8+
9+
use crate::sql_tokens::_helpers::reserved_keywords::is_reserved_keyword;
10+
use crate::sql_tokens::constants::{
11+
CALL_CLOSER, CALL_OPENER, COLUMN_LIST_RELATION_KEYWORDS, CTE_BODY_OPENERS, CTE_BODY_PREFIX,
12+
PLACEHOLDER_PREFIX,
13+
};
14+
use crate::sql_tokens::models::{FingerprintTokenizer, SqlquackFingerprintDialect};
15+
16+
/// Significant token texts with reserved keywords and called names, except `__` placeholders, upper-cased.
17+
pub(crate) fn canonical_tokens(
18+
sql: &str,
19+
dialect: SqlquackFingerprintDialect,
20+
) -> Result<Vec<String>, String> {
21+
let raws: Vec<&str> = significant_texts(sql, dialect.tokenizer)?;
22+
let uppers: Vec<String> = raws.iter().map(|raw| raw.to_ascii_uppercase()).collect();
23+
let names: Vec<bool> = name_positions(&uppers);
24+
let closers: Vec<Option<usize>> = matching_closers(&uppers);
25+
let mut texts: Vec<String> = Vec::with_capacity(raws.len());
26+
for (index, raw) in raws.iter().enumerate() {
27+
let folds = !names[index]
28+
&& is_unquoted_word(raw)
29+
&& (is_reserved_keyword(dialect.keywords, &uppers[index])
30+
|| calls(raw, &uppers, &closers, index));
31+
texts.push(if folds {
32+
uppers[index].clone()
33+
} else {
34+
(*raw).to_owned()
35+
});
36+
}
37+
Ok(texts)
38+
}
39+
40+
/// The text of each significant token of `sql`, in order.
41+
fn significant_texts(sql: &str, tokenizer: FingerprintTokenizer) -> Result<Vec<&str>, String> {
42+
match tokenizer {
43+
FingerprintTokenizer::Lossless(dialect) => Ok(sqlquack_syntax::tokenize(sql, dialect)
44+
.map_err(|error| format!("SQL tokenization failed: {error}"))?
45+
.iter()
46+
.filter(|token| matches!(token.kind, TokenKind::Significant(_)))
47+
.map(|token| token.text(sql))
48+
.collect()),
49+
FingerprintTokenizer::PostgresScanner => scan(sql)
50+
.into_iter()
51+
.map(|token| match token.kind {
52+
PostgresTokenKind::Fault(_) => {
53+
Err(format!("SQL tokenization failed: {}", token.value))
54+
}
55+
_ => sql
56+
.get(token.start..token.end)
57+
.ok_or_else(|| format!("SQL tokenization failed at byte {}", token.start)),
58+
})
59+
.collect(),
60+
}
61+
}
62+
63+
/// The index of the `)` closing each `(`, by the index of the `(`.
64+
fn matching_closers(uppers: &[String]) -> Vec<Option<usize>> {
65+
let mut closers: Vec<Option<usize>> = vec![None; uppers.len()];
66+
let mut open: Vec<usize> = Vec::new();
67+
for (index, token) in uppers.iter().enumerate() {
68+
if token == CALL_OPENER {
69+
open.push(index);
70+
} else if token == CALL_CLOSER
71+
&& let Some(opener) = open.pop()
72+
{
73+
closers[opener] = Some(index);
74+
}
75+
}
76+
closers
77+
}
78+
79+
/// Whether the word at `index` is called: followed by `(` and neither a placeholder nor a relation.
80+
fn calls(raw: &str, uppers: &[String], closers: &[Option<usize>], index: usize) -> bool {
81+
if uppers.get(index + 1).is_none_or(|next| next != CALL_OPENER)
82+
|| raw.starts_with(PLACEHOLDER_PREFIX)
83+
{
84+
return false;
85+
}
86+
let previous: Option<&str> = index.checked_sub(1).map(|before| uppers[before].as_str());
87+
if previous.is_some_and(|word| COLUMN_LIST_RELATION_KEYWORDS.contains(&word)) {
88+
return false;
89+
}
90+
let Some(closer) = closers[index + 1] else {
91+
return true;
92+
};
93+
!(uppers
94+
.get(closer + 1)
95+
.is_some_and(|next| next == CTE_BODY_PREFIX)
96+
&& uppers
97+
.get(closer + 2)
98+
.is_some_and(|next| CTE_BODY_OPENERS.contains(&next.as_str())))
99+
}
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pub(crate) mod canonical_tokens;
2+
pub(crate) mod reserved_keywords;
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//! Per-dialect reserved keywords, which can never be unquoted identifiers in that dialect.
2+
3+
use std::collections::HashSet;
4+
use std::sync::LazyLock;
5+
6+
use crate::sql_tokens::models::KeywordList;
7+
8+
static DUCKDB: LazyLock<HashSet<String>> =
9+
LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/duckdb.txt")));
10+
static POSTGRES: LazyLock<HashSet<String>> =
11+
LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/postgres.txt")));
12+
static SNOWFLAKE: LazyLock<HashSet<String>> =
13+
LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/snowflake.txt")));
14+
static GENERIC: LazyLock<HashSet<String>> =
15+
LazyLock::new(|| keyword_set(include_str!("../reserved_keywords/generic.txt")));
16+
17+
/// Whether the upper-cased word is reserved in `list`.
18+
pub(crate) fn is_reserved_keyword(list: KeywordList, upper_word: &str) -> bool {
19+
let keywords: &HashSet<String> = match list {
20+
KeywordList::DuckDB => &DUCKDB,
21+
KeywordList::PostgreSQL => &POSTGRES,
22+
KeywordList::Snowflake => &SNOWFLAKE,
23+
KeywordList::Generic => &GENERIC,
24+
};
25+
keywords.contains(upper_word)
26+
}
27+
28+
fn keyword_set(data: &str) -> HashSet<String> {
29+
data.lines()
30+
.map(str::trim)
31+
.filter(|line| !line.is_empty() && !line.starts_with('#'))
32+
.map(str::to_ascii_uppercase)
33+
.collect()
34+
}

‎crates/sqlbuild-analysis/src/sql_tokens/constants.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
//! Fingerprint dialects and the function language fingerprinted like a query.
22
3+
use sqlquack_syntax::tokens::models::Dialect;
4+
5+
use crate::sql_tokens::models::{FingerprintTokenizer, KeywordList, SqlquackFingerprintDialect};
6+
37
/// The fingerprint dialect when a project names none.
48
pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic";
59
/// Python's `NATIVE_DIALECT_ALIASES`.
@@ -10,3 +14,48 @@ pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [
1014
];
1115
/// The function language whose definitions are fingerprinted like queries.
1216
pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql";
17+
/// Fingerprint dialects SQLQuack tokenizes; generic uses the DuckDB tokenizer, PostgreSQL its native scanner.
18+
pub(crate) const SQLQUACK_FINGERPRINT_DIALECTS: [SqlquackFingerprintDialect; 6] = [
19+
SqlquackFingerprintDialect {
20+
name: "generic",
21+
tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB),
22+
keywords: KeywordList::Generic,
23+
},
24+
SqlquackFingerprintDialect {
25+
name: "duckdb",
26+
tokenizer: FingerprintTokenizer::Lossless(Dialect::DuckDB),
27+
keywords: KeywordList::DuckDB,
28+
},
29+
SqlquackFingerprintDialect {
30+
name: "postgres",
31+
tokenizer: FingerprintTokenizer::PostgresScanner,
32+
keywords: KeywordList::PostgreSQL,
33+
},
34+
SqlquackFingerprintDialect {
35+
name: "postgresql",
36+
tokenizer: FingerprintTokenizer::PostgresScanner,
37+
keywords: KeywordList::PostgreSQL,
38+
},
39+
SqlquackFingerprintDialect {
40+
name: "snowflake",
41+
tokenizer: FingerprintTokenizer::Lossless(Dialect::Snowflake),
42+
keywords: KeywordList::Snowflake,
43+
},
44+
SqlquackFingerprintDialect {
45+
name: "sqlite",
46+
tokenizer: FingerprintTokenizer::Lossless(Dialect::SQLite),
47+
keywords: KeywordList::Generic,
48+
},
49+
];
50+
/// The token that follows a called function's name.
51+
pub(crate) const CALL_OPENER: &str = "(";
52+
/// The token that closes a call's arguments or a column list.
53+
pub(crate) const CALL_CLOSER: &str = ")";
54+
/// Keywords after which a word followed by `(` names a relation and its column list.
55+
pub(crate) const COLUMN_LIST_RELATION_KEYWORDS: [&str; 2] = ["INTO", "TABLE"];
56+
/// The keyword between a CTE's column list and its body.
57+
pub(crate) const CTE_BODY_PREFIX: &str = "AS";
58+
/// The tokens that can follow `AS` after a CTE's column list.
59+
pub(crate) const CTE_BODY_OPENERS: [&str; 3] = ["(", "MATERIALIZED", "NOT"];
60+
/// The prefix of SQLBuild placeholder calls such as `__ref` and `__source`.
61+
pub(crate) const PLACEHOLDER_PREFIX: &str = "__";

‎crates/sqlbuild-analysis/src/sql_tokens/main/query_fingerprint.rs‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,37 @@ use sha2::{Digest, Sha256};
77

88
use sqlclean::sql_tokens::main::canonical_tokens::canonical_tokens;
99

10+
use crate::sql_tokens::_helpers::canonical_tokens::canonical_tokens as sqlquack_canonical_tokens;
11+
use crate::sql_tokens::constants::SQLQUACK_FINGERPRINT_DIALECTS;
12+
1013
/// Return the SHA-256 hex digest of the canonical token stream of `sql` under `dialect_name`.
1114
pub fn query_fingerprint(sql: &str, dialect_name: &str) -> Result<String, String> {
15+
let texts: Vec<String> = match SQLQUACK_FINGERPRINT_DIALECTS
16+
.iter()
17+
.find(|dialect| dialect.name == dialect_name)
18+
{
19+
Some(dialect) => sqlquack_canonical_tokens(sql, *dialect)?,
20+
None => polyglot_canonical_tokens(sql, dialect_name)?,
21+
};
22+
let mut digest = Sha256::new();
23+
for text in texts {
24+
digest.update(text.len().to_string().as_bytes());
25+
digest.update(b":");
26+
digest.update(text.as_bytes());
27+
}
28+
Ok(format!("{:x}", digest.finalize()))
29+
}
30+
31+
/// Dialects SQLQuack cannot tokenize yet keep SQLClean's Polyglot token stream.
32+
fn polyglot_canonical_tokens(sql: &str, dialect_name: &str) -> Result<Vec<String>, String> {
1233
let dialect_type = DialectType::from_str(dialect_name)
1334
.map_err(|_| format!("unknown SQL fingerprint dialect '{dialect_name}'"))?;
1435
let dialect = Dialect::get(dialect_type);
15-
let mut digest = Sha256::new();
1636
let tokens = dialect
1737
.tokenize(sql)
1838
.map_err(|error| format!("SQL tokenization failed: {error}"))?;
19-
for token in canonical_tokens(sql, &tokens, &dialect)? {
20-
digest.update(token.text.len().to_string().as_bytes());
21-
digest.update(b":");
22-
digest.update(token.text.as_bytes());
23-
}
24-
Ok(format!("{:x}", digest.finalize()))
39+
Ok(canonical_tokens(sql, &tokens, &dialect)?
40+
.into_iter()
41+
.map(|token| token.text)
42+
.collect())
2543
}
Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
//! SQLBuild's query and definition fingerprints over SQLClean's canonical SQL token stream.
1+
//! SQLBuild's query and definition fingerprints over a canonical SQL token stream.
22
3+
mod _helpers;
34
pub mod constants;
45
pub mod main;
6+
pub(crate) mod models;
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
//! How a fingerprint dialect is tokenized and which of its words fold.
2+
3+
use sqlquack_syntax::tokens::models::Dialect;
4+
5+
/// The reserved-keyword list a fingerprint dialect folds.
6+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
7+
pub(crate) enum KeywordList {
8+
DuckDB,
9+
PostgreSQL,
10+
Snowflake,
11+
Generic,
12+
}
13+
14+
/// The SQLQuack tokenizer a fingerprint dialect reads its tokens with.
15+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
16+
pub(crate) enum FingerprintTokenizer {
17+
/// `sqlquack_syntax`'s lossless tokenizer for one dialect.
18+
Lossless(Dialect),
19+
/// The native PostgreSQL scanner of `sqlquack_syntax_postgres`.
20+
PostgresScanner,
21+
}
22+
23+
/// A fingerprint dialect SQLQuack tokenizes: its tokenizer and its reserved keywords.
24+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
25+
pub(crate) struct SqlquackFingerprintDialect {
26+
pub(crate) name: &'static str,
27+
pub(crate) tokenizer: FingerprintTokenizer,
28+
pub(crate) keywords: KeywordList,
29+
}

0 commit comments

Comments
 (0)