Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ pub(crate) fn bigquery_literal_type(expression: &Expression) -> Option<&'static
| Literal::TripleQuotedString(..)
| Literal::RawString(_)
| Literal::JoinedString(..) => Kind::String,
Literal::ByteString(_) => Kind::Bytes,
Literal::ByteString(_) | Literal::RawByteString(_) | Literal::JoinedBytes(..) => {
Kind::Bytes
}
Literal::Date(_) => Kind::Date,
Literal::Datetime(_) => Kind::Datetime,
Literal::Time(_) => Kind::Time,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,50 @@ fn given_one_string_spelled_two_ways_when_grouping_then_the_expressions_match()
sql: "SELECT COALESCE(status, 'ab') AS x FROM orders GROUP BY COALESCE(status, r'ab')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "raw bytes in the select, plain ones in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'ab') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "adjacent bytes in the select, one piece in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS BYTES), b'a' b'b') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "a raw DATE in the select, a plain one in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS DATE), DATE r'2020-01-01') AS x FROM orders GROUP BY COALESCE(CAST(status AS DATE), DATE '2020-01-01')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "an adjacent DATETIME in the select, a plain one in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS DATETIME), DATETIME '2020-01-01' ' 00:00:00') AS x FROM orders GROUP BY COALESCE(CAST(status AS DATETIME), DATETIME '2020-01-01 00:00:00')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "a raw TIMESTAMP in the select, a plain one in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS TIMESTAMP), TIMESTAMP r'2020-01-01 00:00:00') AS x FROM orders GROUP BY COALESCE(CAST(status AS TIMESTAMP), TIMESTAMP '2020-01-01 00:00:00')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "a raw NUMERIC in the select, a plain one in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS NUMERIC), NUMERIC r'1') AS x FROM orders GROUP BY COALESCE(CAST(status AS NUMERIC), NUMERIC '1')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "a raw bytes character in the select, its UTF-8 bytes escaped in GROUP BY",
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'\u{ff}') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'\\xc3\\xbf')",
expected_findings: &[],
},
QueryFindingsTestCase {
description: "raw bytes of another value are still a different expression",
sql: "SELECT COALESCE(CAST(status AS BYTES), rb'ac') AS x FROM orders GROUP BY COALESCE(CAST(status AS BYTES), b'ab')",
expected_findings: &[(
"E230",
"Column must be grouped or used in an aggregate function",
21,
)],
},
QueryFindingsTestCase {
description: "adjacent strings of another value are still a different expression",
sql: "SELECT COALESCE(status, 'a' 'c') AS x FROM orders GROUP BY COALESCE(status, 'ab')",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,11 @@ fn literal_family(literal: &Literal) -> Family {
Literal::Date(_) => Family::Date,
Literal::Time(_) => Family::Time,
Literal::Timestamp(_) | Literal::Datetime(_) => Family::Timestamp,
Literal::HexString(_) | Literal::BitString(_) | Literal::ByteString(_) => Family::Binary,
Literal::HexString(_)
| Literal::BitString(_)
| Literal::ByteString(_)
| Literal::RawByteString(_)
| Literal::JoinedBytes(..) => Family::Binary,
_ => Family::String,
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ use crate::lineage::_helpers::tracing::expressions::is_aggregate;
use crate::lineage::_helpers::tracing::resolution::key;
use crate::lineage::_helpers::tracing::sources::{every_join, from_parts};
use crate::lineage::constants::{
CAST_FIELDS, CUSTOM_TYPE_FIELDS, LITERAL_FIELD, LITERAL_TYPE_FIELD, LITERAL_VALUE_FIELD,
NAME_FIELD, RAW_STRING_KIND, RAW_TYPED_LITERALS, SCOPE_PHASE, SHAPE_COMMENT_SUFFIX,
SHAPE_COSMETIC_FIELDS, SPAN_FIELD, WINDOW_ONLY_FUNCTIONS,
CAST_FIELDS, CUSTOM_TYPE_FIELDS, INFERRED_TYPE_FIELD, LITERAL_FIELD, LITERAL_TYPE_FIELD,
LITERAL_VALUE_FIELD, NAME_FIELD, RAW_STRING_KIND, RAW_TYPED_LITERALS, SCOPE_PHASE,
SHAPE_COMMENT_SUFFIX, SHAPE_COSMETIC_FIELDS, SPAN_FIELD, WINDOW_ONLY_FUNCTIONS,
};
use crate::lineage::models::Engine;
use crate::lineage::models::{LineageContext, LineageScope, NameFinding, ScopeSource};
Expand Down Expand Up @@ -835,7 +835,13 @@ fn spelled_strings(expression: &Expression) -> bool {
matches!(
node,
Expression::Literal(literal)
if matches!(literal.as_ref(), Literal::RawString(_) | Literal::JoinedString(..))
if matches!(
literal.as_ref(),
Literal::RawString(_)
| Literal::JoinedString(..)
| Literal::RawByteString(_)
| Literal::JoinedBytes(..)
)
)
})
}
Expand All @@ -847,11 +853,12 @@ fn plain_strings(value: Value) -> Value {
if let Some(literal) = raw_typed_literal(&map) {
return literal;
}
let spelled = matches!(
map.get(LITERAL_TYPE_FIELD).and_then(Value::as_str),
Some("raw_string" | "joined_string")
);
if spelled {
let plain: Option<&str> = match map.get(LITERAL_TYPE_FIELD).and_then(Value::as_str) {
Some("raw_string" | "joined_string") => Some("string"),
Some("raw_byte_string" | "joined_bytes") => Some("byte_string"),
_ => None,
};
if let Some(plain) = plain {
let text: Option<Value> = match map.remove(LITERAL_VALUE_FIELD) {
Some(Value::Array(mut parts)) if !parts.is_empty() => {
Some(parts.swap_remove(0))
Expand All @@ -860,7 +867,7 @@ fn plain_strings(value: Value) -> Value {
};
map.insert(
LITERAL_TYPE_FIELD.to_owned(),
Value::String("string".to_owned()),
Value::String(plain.to_owned()),
);
map.insert(LITERAL_VALUE_FIELD.to_owned(), text.unwrap_or(Value::Null));
}
Expand All @@ -876,30 +883,39 @@ fn plain_strings(value: Value) -> Value {
}
}

/// The typed literal a serialized cast of a raw string to `DATE`, `TIME`, `TIMESTAMP` or `X` reads as.
/// The typed literal a Databricks or BigQuery typed literal of a raw or adjacent string reads as.
fn raw_typed_literal(map: &Map<String, Value>) -> Option<Value> {
let [cast_field, this_field, to_field] = CAST_FIELDS;
let [type_field, custom] = CUSTOM_TYPE_FIELDS;
let cast: &Map<String, Value> = map.get(cast_field)?.as_object()?;
let text: &Value = cast.get(this_field)?.get(LITERAL_FIELD)?;
let to: &Value = cast.get(to_field)?;
if map.len() != 1
|| text.get(LITERAL_TYPE_FIELD)?.as_str()? != RAW_STRING_KIND
|| to.get(type_field)?.as_str()? != custom
{
let spelled: &str = text.get(LITERAL_TYPE_FIELD)?.as_str()?;
if map.len() != 1 || !matches!(spelled, RAW_STRING_KIND | "joined_string") {
return None;
}
let word: &str = to.get(NAME_FIELD)?.as_str()?;
let (_, kind) = RAW_TYPED_LITERALS.iter().find(|(name, _)| *name == word)?;
let kind: &str = match to.get(type_field)?.as_str()? {
name if name == custom && spelled == RAW_STRING_KIND => {
let word: &str = to.get(NAME_FIELD)?.as_str()?;
RAW_TYPED_LITERALS.iter().find(|(name, _)| *name == word)?.1
}
_ if !cast.contains_key(INFERRED_TYPE_FIELD) => return None,
"date" => "date",
"time" => "time",
"timestamp" if to.get("timezone")?.as_bool()? => "timestamp",
"timestamp" => "datetime",
_ => return None,
};
let value: Value = match text.get(LITERAL_VALUE_FIELD)? {
Value::Array(parts) => parts.first()?.clone(),
value => value.clone(),
};
let mut literal: Map<String, Value> = Map::new();
literal.insert(
LITERAL_TYPE_FIELD.to_owned(),
Value::String((*kind).to_owned()),
);
literal.insert(
LITERAL_VALUE_FIELD.to_owned(),
text.get(LITERAL_VALUE_FIELD)?.clone(),
Value::String(kind.to_owned()),
);
literal.insert(LITERAL_VALUE_FIELD.to_owned(), value);
let mut node: Map<String, Value> = Map::new();
node.insert(LITERAL_FIELD.to_owned(), Value::Object(literal));
Some(Value::Object(node))
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlquack-semantics/src/lineage/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,8 @@ pub(crate) const CAST_FIELDS: [&str; 3] = ["cast", "this", "to"];
pub(crate) const LITERAL_FIELD: &str = "literal";
/// The field a serialized data type records its kind in, and the kind of a type named by a word.
pub(crate) const CUSTOM_TYPE_FIELDS: [&str; 2] = ["data_type", "custom"];
/// The field a serialized cast records the type a BigQuery typed literal gave itself in.
pub(crate) const INFERRED_TYPE_FIELD: &str = "inferred_type";
/// The kind of a serialized raw string literal.
pub(crate) const RAW_STRING_KIND: &str = "raw_string";
/// Typed literal words whose plain string makes a literal of the kind given, a raw one a cast.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ fn principal(name: Identifier, kind: Option<&str>) -> GrantPrincipal {
is_group: false,
is_share: false,
kind: kind.map(str::to_owned),
written: None,
}
}

Expand Down Expand Up @@ -139,9 +140,13 @@ impl Parser<'_> {
if self.string_at(0) {
return match self.string_literal()? {
Expression::Literal(literal) => match *literal {
Literal::String(text)
| Literal::RawString(text)
| Literal::JoinedString(text, _) => Ok(principal(Identifier::new(text), None)),
Literal::String(text) => Ok(principal(Identifier::new(text), None)),
written @ (Literal::RawString(_) | Literal::JoinedString(..)) => {
Ok(GrantPrincipal {
written: Some(written.clone()),
..principal(Identifier::new(written.value_str()), None)
})
}
_ => Err(self.error()),
},
_ => Err(self.error()),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ impl Parser<'_> {
loop {
let token: Token = self.advance()?;
chunks.push(match token.kind {
TokenKind::String {
raw: true,
bytes: true,
} => Literal::RawByteString(token.value),
TokenKind::String { bytes: true, .. } => Literal::ByteString(token.value),
TokenKind::String { raw: true, .. } => Literal::RawString(token.value),
_ => Literal::String(token.value),
});
Expand Down Expand Up @@ -109,17 +114,32 @@ impl Parser<'_> {
}
}

/// A string or bytes literal; one string chunk stays as written, several keep their chunks.
/// A string or bytes literal; one chunk stays as written, several keep their chunks.
pub(crate) fn string_literal(&mut self) -> Parsing<Expression> {
let (mut chunks, bytes) = self.literal_chunks()?;
let text: String = chunks.iter().map(Literal::value_str).collect();
Ok(literal(match (bytes, chunks.len()) {
(true, _) => Literal::ByteString(text),
(false, 1) => chunks.remove(0),
(_, 1) => chunks.remove(0),
(true, _) => Literal::JoinedBytes(text, chunks),
(false, _) => Literal::JoinedString(text, chunks),
}))
}

/// A string literal as written: one plain chunk as its text, else the literal it makes.
fn written_string(&mut self) -> Parsing<Result<String, Literal>> {
let start: usize = self.offset();
let (mut chunks, bytes) = self.literal_chunks()?;
if bytes {
return Err(self.error_with("Syntax error: Unexpected bytes literal", start, true));
}
let text: String = chunks.iter().map(Literal::value_str).collect();
Ok(match (chunks.len(), chunks.first()) {
(1, Some(Literal::String(_))) => Ok(text),
(1, _) => Err(chunks.remove(0)),
_ => Err(Literal::JoinedString(text, chunks)),
})
}

fn primary_base(&mut self) -> Parsing<Expression> {
let Some(token) = self.peek().cloned() else {
return Err(self.error());
Expand Down Expand Up @@ -280,8 +300,8 @@ impl Parser<'_> {
}
"RANGE" if self.char_at(1, '<') => {
let to: DataType = self.data_type()?;
let value: String = self.string_lit()?;
return Ok(typed(string(&value), to));
let text: Result<String, Literal> = self.written_string()?;
return Ok(typed(text.map_or_else(literal, |value| string(&value)), to));
}
"ARRAY" if call_follows && self.starts_query_at(1) => {
self.at += 2;
Expand Down Expand Up @@ -559,22 +579,22 @@ impl Parser<'_> {
})
}

/// A typed literal: a literal node for the date and time types, else a self-typed cast.
/// A typed literal: a literal node for a date or time type of one plain string, else a self-typed cast.
fn typed_literal(&mut self, word: &str) -> Parsing<Expression> {
let type_at: usize = self.at;
self.at += 1;
let value: String = self.string_lit()?;
Ok(match word {
"DATE" => literal(Literal::Date(value)),
"TIME" => literal(Literal::Time(value)),
"DATETIME" => literal(Literal::Datetime(value)),
"TIMESTAMP" => literal(Literal::Timestamp(value)),
_ => {
let text: Result<String, Literal> = self.written_string()?;
Ok(match (word, text) {
("DATE", Ok(value)) => literal(Literal::Date(value)),
("TIME", Ok(value)) => literal(Literal::Time(value)),
("DATETIME", Ok(value)) => literal(Literal::Datetime(value)),
("TIMESTAMP", Ok(value)) => literal(Literal::Timestamp(value)),
(_, text) => {
let end: usize = self.at;
self.at = type_at;
let to: DataType = self.data_type()?;
self.at = end;
typed(string(&value), to)
typed(text.map_or_else(literal, |value| string(&value)), to)
}
})
}
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlquack-syntax-bigquery/src/parse/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,5 @@ mod test_grants;
mod test_parse;
#[path = "tests/test_types.rs"]
mod test_types;
#[path = "tests/test_written_forms.rs"]
mod test_written_forms;
4 changes: 2 additions & 2 deletions crates/sqlquack-syntax-bigquery/src/parse/tests/test_parse.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1065,10 +1065,10 @@ fn given_bytes_literals_when_parsing_then_each_byte_is_one_unit() {
expected_same: false,
},
TreePairTestCase {
description: "a raw bytes literal also reads a character as its UTF-8 bytes",
description: "a raw bytes literal is kept apart from the escaped bytes it reads as",
sql: "SELECT rb'\u{ff}'",
other_sql: "SELECT b'\\xc3\\xbf'",
expected_same: true,
expected_same: false,
},
];
for test_case in test_cases {
Expand Down
Loading
Loading