Skip to content

Commit 9b91fff

Browse files
committed
Merge land/p2-variant-provenance: carry a TO_VARIANT's freedom from JSON null through columns
2 parents e49a550 + 4b776c0 commit 9b91fff

5 files changed

Lines changed: 79 additions & 22 deletions

File tree

‎crates/sqlquack-semantics/src/lineage/_helpers/tracing/expressions.rs‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,8 @@ fn expression_lineage_level(
122122
let target: Option<Rc<str>> =
123123
target.map(|target| mapped_timestamp(Rc::from(target), context));
124124
let nullability: Nullability = match expression {
125-
Expression::Cast(_)
126-
if is_text_type(&cast.to) && variant_of_value(&cast.this, scope, context) =>
127-
{
128-
Nullability::NonNull
125+
Expression::Cast(_) if is_text_type(&cast.to) && inner.json_null_free => {
126+
inner.nullability
129127
}
130128
Expression::Cast(_) => cast_nullability(&inner, target.as_deref(), context),
131129
_ => Nullability::Unknown,
@@ -210,9 +208,17 @@ fn children_lineage(
210208
_ => (child, expression_lineage(child, scope, context)),
211209
})
212210
.collect();
213-
let value_type: ValueType = computed_type(expression, &parts, context);
211+
let (data_type, nullability): ValueType = computed_type(expression, &parts, context);
212+
let json_null_free: bool = json_null_free_variant(expression, &parts, context);
213+
let nullability: Nullability = match (json_null_free, parts.as_slice()) {
214+
(true, [(_, argument)]) => argument.nullability,
215+
_ => nullability,
216+
};
214217
let lineage: Lineage = combined(parts.into_iter().map(|(_, part)| part).collect(), floor);
215-
typed(lineage, value_type)
218+
Lineage {
219+
json_null_free,
220+
..typed(lineage, (data_type, nullability))
221+
}
216222
}
217223

218224
/// Whether `child` is the date part keyword a function like DATEADD or LAST_DAY takes.
@@ -319,26 +325,18 @@ fn is_text_type(data_type: &DataType) -> bool {
319325
)
320326
}
321327

322-
/// Whether `value` is Snowflake's TO_VARIANT of a non-null value no JSON null can hold.
323-
fn variant_of_value(
324-
value: &Expression,
325-
scope: &LineageScope<'_>,
328+
/// Whether `expression` is Snowflake's TO_VARIANT of a value whose type no JSON null can hold.
329+
fn json_null_free_variant(
330+
expression: &Expression,
331+
parts: &[(&Expression, Lineage)],
326332
context: LineageContext<'_>,
327333
) -> bool {
328-
let Expression::Function(function) = value else {
334+
let (Expression::Function(function), [(_, argument)]) = (expression, parts) else {
329335
return false;
330336
};
331-
let [argument] = function.args.as_slice() else {
332-
return false;
333-
};
334-
if context.profile.name() != SNOWFLAKE_PROFILE
335-
|| function.quoted
336-
|| !function.name.eq_ignore_ascii_case(SNOWFLAKE_TO_VARIANT)
337-
{
338-
return false;
339-
}
340-
let argument: Lineage = expression_lineage(argument, scope, context);
341-
argument.nullability == Nullability::NonNull
337+
context.profile.name() == SNOWFLAKE_PROFILE
338+
&& !function.quoted
339+
&& function.name.eq_ignore_ascii_case(SNOWFLAKE_TO_VARIANT)
342340
&& argument.data_type.as_deref().is_some_and(|spelling| {
343341
!SNOWFLAKE_JSON_HOLDERS.contains(&canonical_type(spelling).as_str())
344342
})

‎crates/sqlquack-semantics/src/lineage/_helpers/tracing/lineages.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub(crate) fn constant() -> Lineage {
2626
rows_lost: false,
2727
data_type: None,
2828
nullability: Nullability::Unknown,
29+
json_null_free: false,
2930
}
3031
}
3132

@@ -85,6 +86,7 @@ pub(crate) fn combined(parts: Vec<Lineage>, floor: TransformKind) -> Lineage {
8586
rows_lost,
8687
data_type: None,
8788
nullability: Nullability::Unknown,
89+
json_null_free: false,
8890
}
8991
}
9092

@@ -205,6 +207,7 @@ pub(crate) fn merged(parts: Vec<Lineage>, profile: &dyn DialectProfile) -> Linea
205207
rows_lost,
206208
data_type,
207209
nullability,
210+
json_null_free: false,
208211
}
209212
}
210213

‎crates/sqlquack-semantics/src/lineage/models.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ pub(crate) struct Lineage {
133133
pub(crate) data_type: Option<Rc<str>>,
134134
/// Whether the value may be NULL.
135135
pub(crate) nullability: Nullability,
136+
/// Whether the value is a VARIANT that cannot hold a JSON null, as TO_VARIANT of a non-VARIANT.
137+
pub(crate) json_null_free: bool,
136138
}
137139

138140
/// One column a FROM item offers, with its lineage.

‎crates/sqlquack-semantics/src/lineage/tests/helpers.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@ pub(crate) fn typed_shop_schema() -> TypedShopSchema {
256256
("TO_JSON".to_owned(), "TEXT".to_owned()),
257257
("LISTAGG".to_owned(), "TEXT".to_owned()),
258258
("TO_VARIANT".to_owned(), "VARIANT".to_owned()),
259+
("PARSE_JSON".to_owned(), "VARIANT".to_owned()),
259260
]),
260261
rules: HashMap::from([
261262
("IFF".to_owned(), NullabilityRule::ConditionalResult),

‎crates/sqlquack-semantics/src/lineage/tests/test_recorded_types.rs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,59 @@ fn given_a_query_when_typing_outputs_then_nulls_and_numbers_follow_the_recorded_
186186
("e", Some("INT"), Nullable),
187187
],
188188
},
189+
DialectTypeTestCase {
190+
description: "TO_VARIANT of each non-JSON family casts to text as null as its value",
191+
sql: "SELECT TO_VARIANT(c.name)::VARCHAR AS text, TO_VARIANT('null')::VARCHAR AS null_word, \
192+
TO_VARIANT(c.id)::VARCHAR AS number, TO_VARIANT(1.5::FLOAT)::VARCHAR AS float, \
193+
TO_VARIANT(TRUE)::VARCHAR AS boolean, TO_VARIANT(s.shipped_on)::VARCHAR AS date, \
194+
TO_VARIANT(s.cutoff)::VARCHAR AS time, TO_VARIANT(s.shipped_at)::VARCHAR AS ts, \
195+
TO_VARIANT(o.amount)::VARCHAR AS nullable_number \
196+
FROM customers c, shipments s, orders o",
197+
dialect: Dialect::Snowflake,
198+
profile: &SNOWFLAKE,
199+
expected_outputs: &[
200+
("text", Some("VARCHAR"), NonNull),
201+
("null_word", Some("VARCHAR"), NonNull),
202+
("number", Some("VARCHAR"), NonNull),
203+
("float", Some("VARCHAR"), NonNull),
204+
("boolean", Some("VARCHAR"), NonNull),
205+
("date", Some("VARCHAR"), NonNull),
206+
("time", Some("VARCHAR"), NonNull),
207+
("ts", Some("VARCHAR"), NonNull),
208+
("nullable_number", Some("VARCHAR"), Nullable),
209+
],
210+
},
211+
DialectTypeTestCase {
212+
description: "TO_VARIANT of a VARIANT, which may be a JSON null, or a path into one, may cast to NULL; \
213+
a whole OBJECT or ARRAY never is a JSON null",
214+
sql: "SELECT TO_VARIANT(e.payload)::VARCHAR AS variant, \
215+
TO_VARIANT(PARSE_JSON('null'))::VARCHAR AS parsed, \
216+
TO_VARIANT(c.name):a::VARCHAR AS path, \
217+
TO_VARIANT('{}'::OBJECT)::VARCHAR AS object, TO_VARIANT('[]'::ARRAY)::VARCHAR AS array \
218+
FROM customers c, events e",
219+
dialect: Dialect::Snowflake,
220+
profile: &SNOWFLAKE,
221+
expected_outputs: &[
222+
("variant", Some("VARCHAR"), Nullable),
223+
("parsed", Some("VARCHAR"), Nullable),
224+
("path", Some("VARCHAR"), Unknown),
225+
("object", Some("VARCHAR"), NonNull),
226+
("array", Some("VARCHAR"), NonNull),
227+
],
228+
},
229+
DialectTypeTestCase {
230+
description: "a TO_VARIANT column of a CTE, filtered IS NOT NULL, casts to non-null text",
231+
sql: "WITH d AS (SELECT DISTINCT TO_VARIANT(o.status) AS tag_value FROM orders o), \
232+
p AS (SELECT DISTINCT PARSE_JSON(o.status) AS tag_value FROM orders o) \
233+
SELECT d.tag_value::VARCHAR AS tag_text, p.tag_value::VARCHAR AS parsed_text \
234+
FROM d, p WHERE d.tag_value IS NOT NULL AND p.tag_value IS NOT NULL",
235+
dialect: Dialect::Snowflake,
236+
profile: &SNOWFLAKE,
237+
expected_outputs: &[
238+
("tag_text", Some("VARCHAR"), NonNull),
239+
("parsed_text", Some("VARCHAR"), Nullable),
240+
],
241+
},
189242
DialectTypeTestCase {
190243
description: "DATEDIFF takes its unit's recorded NUMBER, and TIMEs their own",
191244
sql: "SELECT DATEDIFF(day, shipped_on, shipped_at) AS a, \

0 commit comments

Comments
 (0)