Skip to content

Commit 61a63a7

Browse files
committed
fix(bigquery)!: keep raw and adjacent pieces of bytes, typed literals and grantees
BigQuery gave one tree for literals written different ways: b'ab', b'a' b'b' and rb'ab'; DATE, DATETIME, TIME and TIMESTAMP of a plain, a raw or an adjacent string; JSON, NUMERIC and the other self-typed casts of a raw or adjacent string; and a GRANT grantee as a plain, raw or adjacent string. The shared tree gains two literal kinds, raw bytes and adjacent bytes, each keeping what was written. A typed literal of a raw or adjacent string is a self-typed cast of that literal, and a grantee written raw or in pieces records the written literal beside its name. The GROUP BY match that already treats r'ab' as 'ab' now also reads raw and adjacent bytes as one bytes literal and a raw or adjacent date or time typed literal as the plain one, so these spellings give no new E230. Bytes of the new kinds type as BYTES. BREAKING CHANGE: sqlquack-syntax adds Literal::RawByteString and Literal::JoinedBytes, and GrantPrincipal gains a written field, so an exhaustive match on Literal or a GrantPrincipal struct literal no longer compiles.
1 parent 4dd6098 commit 61a63a7

16 files changed

Lines changed: 232 additions & 46 deletions

File tree

‎crates/sqlquack-semantics/src/bigquery/profile/_helpers/typing.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ pub(crate) fn bigquery_literal_type(expression: &Expression) -> Option<&'static
9595
| Literal::TripleQuotedString(..)
9696
| Literal::RawString(_)
9797
| Literal::JoinedString(..) => Kind::String,
98-
Literal::ByteString(_) => Kind::Bytes,
98+
Literal::ByteString(_) | Literal::RawByteString(_) | Literal::JoinedBytes(..) => {
99+
Kind::Bytes
100+
}
99101
Literal::Date(_) => Kind::Date,
100102
Literal::Datetime(_) => Kind::Datetime,
101103
Literal::Time(_) => Kind::Time,

‎crates/sqlquack-semantics/src/bigquery/profile/tests/test_checks.rs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,50 @@ fn given_one_string_spelled_two_ways_when_grouping_then_the_expressions_match()
343343
sql: "SELECT COALESCE(status, 'ab') AS x FROM orders GROUP BY COALESCE(status, r'ab')",
344344
expected_findings: &[],
345345
},
346+
QueryFindingsTestCase {
347+
description: "raw bytes in the select, plain ones in GROUP BY",
348+
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'ab') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
349+
expected_findings: &[],
350+
},
351+
QueryFindingsTestCase {
352+
description: "adjacent bytes in the select, one piece in GROUP BY",
353+
sql: "SELECT COALESCE(CAST(status AS BYTES), b'a' b'b') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
354+
expected_findings: &[],
355+
},
356+
QueryFindingsTestCase {
357+
description: "a raw DATE in the select, a plain one in GROUP BY",
358+
sql: "SELECT COALESCE(CAST(status AS DATE), DATE r'2020-01-01') AS x FROM orders GROUP BY COALESCE(CAST(status AS DATE), DATE '2020-01-01')",
359+
expected_findings: &[],
360+
},
361+
QueryFindingsTestCase {
362+
description: "an adjacent DATETIME in the select, a plain one in GROUP BY",
363+
sql: "SELECT COALESCE(CAST(status AS DATETIME), DATETIME '2020-01-01' ' 00:00:00') AS x FROM orders GROUP BY COALESCE(CAST(status AS DATETIME), DATETIME '2020-01-01 00:00:00')",
364+
expected_findings: &[],
365+
},
366+
QueryFindingsTestCase {
367+
description: "a raw TIMESTAMP in the select, a plain one in GROUP BY",
368+
sql: "SELECT COALESCE(CAST(status AS TIMESTAMP), TIMESTAMP r'2020-01-01 00:00:00') AS x FROM orders GROUP BY COALESCE(CAST(status AS TIMESTAMP), TIMESTAMP '2020-01-01 00:00:00')",
369+
expected_findings: &[],
370+
},
371+
QueryFindingsTestCase {
372+
description: "a raw NUMERIC in the select, a plain one in GROUP BY",
373+
sql: "SELECT COALESCE(CAST(status AS NUMERIC), NUMERIC r'1') AS x FROM orders GROUP BY COALESCE(CAST(status AS NUMERIC), NUMERIC '1')",
374+
expected_findings: &[],
375+
},
376+
QueryFindingsTestCase {
377+
description: "a raw bytes character in the select, its UTF-8 bytes escaped in GROUP BY",
378+
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'\u{ff}') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'\\xc3\\xbf')",
379+
expected_findings: &[],
380+
},
381+
QueryFindingsTestCase {
382+
description: "raw bytes of another value are still a different expression",
383+
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'ac') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
384+
expected_findings: &[(
385+
"E230",
386+
"Column must be grouped or used in an aggregate function",
387+
21,
388+
)],
389+
},
346390
QueryFindingsTestCase {
347391
description: "adjacent strings of another value are still a different expression",
348392
sql: "SELECT COALESCE(status, 'a' 'c') AS x FROM orders GROUP BY COALESCE(status, 'ab')",

‎crates/sqlquack-semantics/src/lineage/_helpers/checks/families.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -242,7 +242,11 @@ fn literal_family(literal: &Literal) -> Family {
242242
Literal::Date(_) => Family::Date,
243243
Literal::Time(_) => Family::Time,
244244
Literal::Timestamp(_) | Literal::Datetime(_) => Family::Timestamp,
245-
Literal::HexString(_) | Literal::BitString(_) | Literal::ByteString(_) => Family::Binary,
245+
Literal::HexString(_)
246+
| Literal::BitString(_)
247+
| Literal::ByteString(_)
248+
| Literal::RawByteString(_)
249+
| Literal::JoinedBytes(..) => Family::Binary,
246250
_ => Family::String,
247251
}
248252
}

‎crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs‎

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ use crate::lineage::_helpers::tracing::expressions::is_aggregate;
1515
use crate::lineage::_helpers::tracing::resolution::key;
1616
use crate::lineage::_helpers::tracing::sources::{every_join, from_parts};
1717
use crate::lineage::constants::{
18-
CAST_FIELDS, CUSTOM_TYPE_FIELDS, LITERAL_FIELD, LITERAL_TYPE_FIELD, LITERAL_VALUE_FIELD,
19-
NAME_FIELD, RAW_STRING_KIND, RAW_TYPED_LITERALS, SCOPE_PHASE, SHAPE_COMMENT_SUFFIX,
20-
SHAPE_COSMETIC_FIELDS, SPAN_FIELD, WINDOW_ONLY_FUNCTIONS,
18+
CAST_FIELDS, CUSTOM_TYPE_FIELDS, INFERRED_TYPE_FIELD, LITERAL_FIELD, LITERAL_TYPE_FIELD,
19+
LITERAL_VALUE_FIELD, NAME_FIELD, RAW_STRING_KIND, RAW_TYPED_LITERALS, SCOPE_PHASE,
20+
SHAPE_COMMENT_SUFFIX, SHAPE_COSMETIC_FIELDS, SPAN_FIELD, WINDOW_ONLY_FUNCTIONS,
2121
};
2222
use crate::lineage::models::Engine;
2323
use crate::lineage::models::{LineageContext, LineageScope, NameFinding, ScopeSource};
@@ -835,7 +835,13 @@ fn spelled_strings(expression: &Expression) -> bool {
835835
matches!(
836836
node,
837837
Expression::Literal(literal)
838-
if matches!(literal.as_ref(), Literal::RawString(_) | Literal::JoinedString(..))
838+
if matches!(
839+
literal.as_ref(),
840+
Literal::RawString(_)
841+
| Literal::JoinedString(..)
842+
| Literal::RawByteString(_)
843+
| Literal::JoinedBytes(..)
844+
)
839845
)
840846
})
841847
}
@@ -847,11 +853,12 @@ fn plain_strings(value: Value) -> Value {
847853
if let Some(literal) = raw_typed_literal(&map) {
848854
return literal;
849855
}
850-
let spelled = matches!(
851-
map.get(LITERAL_TYPE_FIELD).and_then(Value::as_str),
852-
Some("raw_string" | "joined_string")
853-
);
854-
if spelled {
856+
let plain: Option<&str> = match map.get(LITERAL_TYPE_FIELD).and_then(Value::as_str) {
857+
Some("raw_string" | "joined_string") => Some("string"),
858+
Some("raw_byte_string" | "joined_bytes") => Some("byte_string"),
859+
_ => None,
860+
};
861+
if let Some(plain) = plain {
855862
let text: Option<Value> = match map.remove(LITERAL_VALUE_FIELD) {
856863
Some(Value::Array(mut parts)) if !parts.is_empty() => {
857864
Some(parts.swap_remove(0))
@@ -860,7 +867,7 @@ fn plain_strings(value: Value) -> Value {
860867
};
861868
map.insert(
862869
LITERAL_TYPE_FIELD.to_owned(),
863-
Value::String("string".to_owned()),
870+
Value::String(plain.to_owned()),
864871
);
865872
map.insert(LITERAL_VALUE_FIELD.to_owned(), text.unwrap_or(Value::Null));
866873
}
@@ -876,30 +883,39 @@ fn plain_strings(value: Value) -> Value {
876883
}
877884
}
878885

879-
/// The typed literal a serialized cast of a raw string to `DATE`, `TIME`, `TIMESTAMP` or `X` reads as.
886+
/// The typed literal a Databricks or BigQuery typed literal of a raw or adjacent string reads as.
880887
fn raw_typed_literal(map: &Map<String, Value>) -> Option<Value> {
881888
let [cast_field, this_field, to_field] = CAST_FIELDS;
882889
let [type_field, custom] = CUSTOM_TYPE_FIELDS;
883890
let cast: &Map<String, Value> = map.get(cast_field)?.as_object()?;
884891
let text: &Value = cast.get(this_field)?.get(LITERAL_FIELD)?;
885892
let to: &Value = cast.get(to_field)?;
886-
if map.len() != 1
887-
|| text.get(LITERAL_TYPE_FIELD)?.as_str()? != RAW_STRING_KIND
888-
|| to.get(type_field)?.as_str()? != custom
889-
{
893+
let spelled: &str = text.get(LITERAL_TYPE_FIELD)?.as_str()?;
894+
if map.len() != 1 || !matches!(spelled, RAW_STRING_KIND | "joined_string") {
890895
return None;
891896
}
892-
let word: &str = to.get(NAME_FIELD)?.as_str()?;
893-
let (_, kind) = RAW_TYPED_LITERALS.iter().find(|(name, _)| *name == word)?;
897+
let kind: &str = match to.get(type_field)?.as_str()? {
898+
name if name == custom && spelled == RAW_STRING_KIND => {
899+
let word: &str = to.get(NAME_FIELD)?.as_str()?;
900+
RAW_TYPED_LITERALS.iter().find(|(name, _)| *name == word)?.1
901+
}
902+
_ if !cast.contains_key(INFERRED_TYPE_FIELD) => return None,
903+
"date" => "date",
904+
"time" => "time",
905+
"timestamp" if to.get("timezone")?.as_bool()? => "timestamp",
906+
"timestamp" => "datetime",
907+
_ => return None,
908+
};
909+
let value: Value = match text.get(LITERAL_VALUE_FIELD)? {
910+
Value::Array(parts) => parts.first()?.clone(),
911+
value => value.clone(),
912+
};
894913
let mut literal: Map<String, Value> = Map::new();
895914
literal.insert(
896915
LITERAL_TYPE_FIELD.to_owned(),
897-
Value::String((*kind).to_owned()),
898-
);
899-
literal.insert(
900-
LITERAL_VALUE_FIELD.to_owned(),
901-
text.get(LITERAL_VALUE_FIELD)?.clone(),
916+
Value::String(kind.to_owned()),
902917
);
918+
literal.insert(LITERAL_VALUE_FIELD.to_owned(), value);
903919
let mut node: Map<String, Value> = Map::new();
904920
node.insert(LITERAL_FIELD.to_owned(), Value::Object(literal));
905921
Some(Value::Object(node))

‎crates/sqlquack-semantics/src/lineage/constants.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,8 @@ pub(crate) const CAST_FIELDS: [&str; 3] = ["cast", "this", "to"];
248248
pub(crate) const LITERAL_FIELD: &str = "literal";
249249
/// The field a serialized data type records its kind in, and the kind of a type named by a word.
250250
pub(crate) const CUSTOM_TYPE_FIELDS: [&str; 2] = ["data_type", "custom"];
251+
/// The field a serialized cast records the type a BigQuery typed literal gave itself in.
252+
pub(crate) const INFERRED_TYPE_FIELD: &str = "inferred_type";
251253
/// The kind of a serialized raw string literal.
252254
pub(crate) const RAW_STRING_KIND: &str = "raw_string";
253255
/// Typed literal words whose plain string makes a literal of the kind given, a raw one a cast.

‎crates/sqlquack-syntax-bigquery/src/parse/_helpers/commands/grants.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ fn principal(name: Identifier, kind: Option<&str>) -> GrantPrincipal {
2626
is_group: false,
2727
is_share: false,
2828
kind: kind.map(str::to_owned),
29+
written: None,
2930
}
3031
}
3132

@@ -139,9 +140,13 @@ impl Parser<'_> {
139140
if self.string_at(0) {
140141
return match self.string_literal()? {
141142
Expression::Literal(literal) => match *literal {
142-
Literal::String(text)
143-
| Literal::RawString(text)
144-
| Literal::JoinedString(text, _) => Ok(principal(Identifier::new(text), None)),
143+
Literal::String(text) => Ok(principal(Identifier::new(text), None)),
144+
written @ (Literal::RawString(_) | Literal::JoinedString(..)) => {
145+
Ok(GrantPrincipal {
146+
written: Some(written.clone()),
147+
..principal(Identifier::new(written.value_str()), None)
148+
})
149+
}
145150
_ => Err(self.error()),
146151
},
147152
_ => Err(self.error()),

‎crates/sqlquack-syntax-bigquery/src/parse/_helpers/values/primary.rs‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ impl Parser<'_> {
7373
loop {
7474
let token: Token = self.advance()?;
7575
chunks.push(match token.kind {
76+
TokenKind::String {
77+
raw: true,
78+
bytes: true,
79+
} => Literal::RawByteString(token.value),
80+
TokenKind::String { bytes: true, .. } => Literal::ByteString(token.value),
7681
TokenKind::String { raw: true, .. } => Literal::RawString(token.value),
7782
_ => Literal::String(token.value),
7883
});
@@ -109,17 +114,32 @@ impl Parser<'_> {
109114
}
110115
}
111116

112-
/// A string or bytes literal; one string chunk stays as written, several keep their chunks.
117+
/// A string or bytes literal; one chunk stays as written, several keep their chunks.
113118
pub(crate) fn string_literal(&mut self) -> Parsing<Expression> {
114119
let (mut chunks, bytes) = self.literal_chunks()?;
115120
let text: String = chunks.iter().map(Literal::value_str).collect();
116121
Ok(literal(match (bytes, chunks.len()) {
117-
(true, _) => Literal::ByteString(text),
118-
(false, 1) => chunks.remove(0),
122+
(_, 1) => chunks.remove(0),
123+
(true, _) => Literal::JoinedBytes(text, chunks),
119124
(false, _) => Literal::JoinedString(text, chunks),
120125
}))
121126
}
122127

128+
/// A string literal as written: one plain chunk as its text, else the literal it makes.
129+
fn written_string(&mut self) -> Parsing<Result<String, Literal>> {
130+
let start: usize = self.offset();
131+
let (mut chunks, bytes) = self.literal_chunks()?;
132+
if bytes {
133+
return Err(self.error_with("Syntax error: Unexpected bytes literal", start, true));
134+
}
135+
let text: String = chunks.iter().map(Literal::value_str).collect();
136+
Ok(match (chunks.len(), chunks.first()) {
137+
(1, Some(Literal::String(_))) => Ok(text),
138+
(1, _) => Err(chunks.remove(0)),
139+
_ => Err(Literal::JoinedString(text, chunks)),
140+
})
141+
}
142+
123143
fn primary_base(&mut self) -> Parsing<Expression> {
124144
let Some(token) = self.peek().cloned() else {
125145
return Err(self.error());
@@ -280,8 +300,8 @@ impl Parser<'_> {
280300
}
281301
"RANGE" if self.char_at(1, '<') => {
282302
let to: DataType = self.data_type()?;
283-
let value: String = self.string_lit()?;
284-
return Ok(typed(string(&value), to));
303+
let text: Result<String, Literal> = self.written_string()?;
304+
return Ok(typed(text.map_or_else(literal, |value| string(&value)), to));
285305
}
286306
"ARRAY" if call_follows && self.starts_query_at(1) => {
287307
self.at += 2;
@@ -559,22 +579,22 @@ impl Parser<'_> {
559579
})
560580
}
561581

562-
/// A typed literal: a literal node for the date and time types, else a self-typed cast.
582+
/// A typed literal: a literal node for a date or time type of one plain string, else a self-typed cast.
563583
fn typed_literal(&mut self, word: &str) -> Parsing<Expression> {
564584
let type_at: usize = self.at;
565585
self.at += 1;
566-
let value: String = self.string_lit()?;
567-
Ok(match word {
568-
"DATE" => literal(Literal::Date(value)),
569-
"TIME" => literal(Literal::Time(value)),
570-
"DATETIME" => literal(Literal::Datetime(value)),
571-
"TIMESTAMP" => literal(Literal::Timestamp(value)),
572-
_ => {
586+
let text: Result<String, Literal> = self.written_string()?;
587+
Ok(match (word, text) {
588+
("DATE", Ok(value)) => literal(Literal::Date(value)),
589+
("TIME", Ok(value)) => literal(Literal::Time(value)),
590+
("DATETIME", Ok(value)) => literal(Literal::Datetime(value)),
591+
("TIMESTAMP", Ok(value)) => literal(Literal::Timestamp(value)),
592+
(_, text) => {
573593
let end: usize = self.at;
574594
self.at = type_at;
575595
let to: DataType = self.data_type()?;
576596
self.at = end;
577-
typed(string(&value), to)
597+
typed(text.map_or_else(literal, |value| string(&value)), to)
578598
}
579599
})
580600
}

‎crates/sqlquack-syntax-bigquery/src/parse/tests.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ mod test_grants;
66
mod test_parse;
77
#[path = "tests/test_types.rs"]
88
mod test_types;
9+
#[path = "tests/test_written_forms.rs"]
10+
mod test_written_forms;

‎crates/sqlquack-syntax-bigquery/src/parse/tests/test_parse.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1065,10 +1065,10 @@ fn given_bytes_literals_when_parsing_then_each_byte_is_one_unit() {
10651065
expected_same: false,
10661066
},
10671067
TreePairTestCase {
1068-
description: "a raw bytes literal also reads a character as its UTF-8 bytes",
1068+
description: "a raw bytes literal is kept apart from the escaped bytes it reads as",
10691069
sql: "SELECT rb'\u{ff}'",
10701070
other_sql: "SELECT b'\\xc3\\xbf'",
1071-
expected_same: true,
1071+
expected_same: false,
10721072
},
10731073
];
10741074
for test_case in test_cases {

0 commit comments

Comments
 (0)