Skip to content

Commit 6352fa6

Browse files
authored
feat: Python table functions on Snowflake (#1610)
* feat: Python table functions on Snowflake Add @table_fn(...) for whole-partition Python table functions. Each call receives one partition as a data frame, polars by default or pandas from a pd.DataFrame annotation or the [defaults] function_dataframe setting, and returns a data frame matching its declared returns columns. Calls must use OVER (PARTITION BY ...), checked at compile time in models, function bodies and SQL tests. Polars functions reject fractional NUMBER arguments. Snowflake deploys a vectorized end_partition handler that checks the returned columns. * fix: harden Python table function partition parsing and handler Comments before OVER and inside its clause no longer hide the partition. The generated Snowflake handler prefixes its locals so any entry point name works, and names repeated returned columns. DEC counts as a fixed-point type for polars arguments. The docs state how the frame annotation is read. * fix: skip dialect line comments before a table function's OVER clause The partition check, analysis, SQL-test markers and Snowflake rendering now skip each dialect's own line comments, such as Snowflake's //, before OVER and inside it. * refactor: share the temporary function copy between Snowflake and Databricks * test: type the stubbed Snowflake module in the table function handler test * test: list function_dataframe in the unknown default key golden
1 parent 0707d18 commit 6352fa6

114 files changed

Lines changed: 2269 additions & 497 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ use regex::{Captures, Regex};
66

77
use crate::compiler::_helpers::sql_tests::planning::compile_error;
88
use crate::relation_markers::main::lateral_start::lateral_start;
9+
use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open;
910
use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
1011
use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
1112
use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
1213
use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed};
1314

1415
pub(crate) fn replace_callable_markers<F>(
1516
sql: &str,
16-
(pattern, drop_lateral): (&Regex, bool),
17+
(pattern, drop_lateral, table_call): (&Regex, bool, bool),
1718
syntax: &LexicalSyntax,
1819
mut replacement: F,
1920
) -> Result<(String, HashSet<String>), String>
@@ -35,7 +36,13 @@ where
3536
if !sql[suffix_start..].starts_with('(') {
3637
continue;
3738
}
38-
let suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
39+
let mut suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
40+
if table_call
41+
&& let Some(open) = over_clause_open(sql, suffix_end, &syntax.line_comment_prefixes)
42+
&& !protected.contains(open)
43+
{
44+
suffix_end = matching_paren_end(sql, open, syntax)?;
45+
}
3946
let Some(name) = captures.get(1).map(|value| value.as_str()) else {
4047
continue;
4148
};

‎crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,7 +1709,7 @@ fn resolve_table_function_fixtures<'a>(
17091709
}
17101710
let (replaced, reached) = replace_callable_markers(
17111711
sql,
1712-
(&patterns.table_function, true),
1712+
(&patterns.table_function, true, true),
17131713
&patterns.lexical,
17141714
|name, _call_suffix| {
17151715
fixtures
@@ -1750,7 +1750,7 @@ fn resolve_function_calls(
17501750
};
17511751
let (result, _) = replace_callable_markers(
17521752
sql,
1753-
(pattern, false),
1753+
(pattern, false, table_function),
17541754
&patterns.lexical,
17551755
|name, call_suffix| {
17561756
let function = functions.get(name)?;

‎crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,31 +55,57 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la
5555
description: "a mock after a comma LATERAL stands alone",
5656
sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l",
5757
drop_lateral: true,
58-
expected_sql: "SELECT l.a FROM b, mock_f AS l",
58+
table_call: true,
59+
expected_sql: "SELECT l.a FROM b, mock_f[(b.id)] AS l",
5960
},
6061
CallableMarkerTestCase {
6162
description: "a mock after JOIN lateral, in any case, stands alone",
6263
sql: "SELECT l.a FROM b JOIN lateral __table_fn(\"f\")(b.id) AS l ON TRUE",
6364
drop_lateral: true,
64-
expected_sql: "SELECT l.a FROM b JOIN mock_f AS l ON TRUE",
65+
table_call: true,
66+
expected_sql: "SELECT l.a FROM b JOIN mock_f[(b.id)] AS l ON TRUE",
6567
},
6668
CallableMarkerTestCase {
6769
description: "a rendered call keeps its LATERAL",
6870
sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l",
6971
drop_lateral: false,
70-
expected_sql: "SELECT l.a FROM b, LATERAL mock_f AS l",
72+
table_call: true,
73+
expected_sql: "SELECT l.a FROM b, LATERAL mock_f[(b.id)] AS l",
7174
},
7275
CallableMarkerTestCase {
7376
description: "lateral ending a comment before the call is kept",
7477
sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l",
7578
drop_lateral: true,
76-
expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f AS l",
79+
table_call: true,
80+
expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f[(1)] AS l",
7781
},
7882
CallableMarkerTestCase {
7983
description: "a word ending in lateral before the call is kept",
8084
sql: "SELECT l.a FROM collateral, __table_fn(\"f\")(1) AS l",
8185
drop_lateral: true,
82-
expected_sql: "SELECT l.a FROM collateral, mock_f AS l",
86+
table_call: true,
87+
expected_sql: "SELECT l.a FROM collateral, mock_f[(1)] AS l",
88+
},
89+
CallableMarkerTestCase {
90+
description: "a table function call's suffix takes its OVER clause with it",
91+
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id) OVER (PARTITION BY (b.x)) AS l",
92+
drop_lateral: true,
93+
table_call: true,
94+
expected_sql: "SELECT l.a FROM b, mock_f[(b.id) OVER (PARTITION BY (b.x))] AS l",
95+
},
96+
CallableMarkerTestCase {
97+
description: "an OVER clause in any case and across lines belongs to the call",
98+
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (partition by b.x) l",
99+
drop_lateral: false,
100+
table_call: true,
101+
expected_sql: "SELECT l.a FROM b, mock_f[(b.id)\n over (partition by b.x)] l",
102+
},
103+
CallableMarkerTestCase {
104+
description: "a scalar call keeps a window clause after it",
105+
sql: "SELECT __table_fn(\"f\")(b.id) OVER (ORDER BY b.x) FROM b",
106+
drop_lateral: false,
107+
table_call: false,
108+
expected_sql: "SELECT mock_f[(b.id)] OVER (ORDER BY b.x) FROM b",
83109
},
84110
];
85111
let pattern = Regex::new(r#"(?i)__table_fn\("([^"]+)"\)"#).expect("valid marker pattern");
@@ -90,9 +116,9 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la
90116
for test_case in test_cases {
91117
let replaced = replace_callable_markers(
92118
test_case.sql,
93-
(&pattern, test_case.drop_lateral),
119+
(&pattern, test_case.drop_lateral, test_case.table_call),
94120
&syntax,
95-
|name, _| Some((format!("mock_{name}"), true)),
121+
|name, suffix| Some((format!("mock_{name}[{suffix}]"), true)),
96122
)
97123
.map(|(sql, _)| sql);
98124
assert_eq!(

‎crates/sqlbuild-analysis/src/compiler/tests/test_types.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,12 @@ pub(crate) struct MarkerNamesTestCase {
6363
pub(crate) expected_replaced_sql: &'static str,
6464
}
6565

66-
/// A table function call replaced by a name, with or without dropping a `LATERAL` before it.
66+
/// A call replaced by a name or a rendered call, with or without its `LATERAL` and `OVER (...)`.
6767
pub(crate) struct CallableMarkerTestCase {
6868
pub(crate) description: &'static str,
6969
pub(crate) sql: &'static str,
7070
pub(crate) drop_lateral: bool,
71+
pub(crate) table_call: bool,
7172
pub(crate) expected_sql: &'static str,
7273
}
7374

‎crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
33
use crate::planner_identity::models::FunctionFingerprintFields;
44

5-
/// The language, signature, runtime fields and body, one field per line.
5+
/// The language, signature, runtime fields, any frame library and body, one field per line.
66
pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String {
77
let render = |pairs: &[(String, String)]| -> String {
88
pairs
@@ -11,16 +11,21 @@ pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String {
1111
.collect::<Vec<String>>()
1212
.join(",")
1313
};
14-
[
14+
let mut lines: Vec<String> = vec![
1515
format!("language={}", fields.language),
1616
format!("arguments={}", render(&fields.arguments)),
1717
format!("returns={}", fields.returns),
1818
format!("return_columns={}", render(&fields.return_columns)),
1919
format!("runtime_version={}", fields.runtime_version),
2020
format!("entry_point={}", fields.entry_point),
2121
format!("packages={}", fields.packages.join(",")),
22-
"body=".to_owned(),
23-
fields.body_sql.clone(),
24-
]
25-
.join("\n")
22+
];
23+
lines.extend(
24+
fields
25+
.dataframe
26+
.as_ref()
27+
.map(|dataframe| format!("dataframe={dataframe}")),
28+
);
29+
lines.extend(["body=".to_owned(), fields.body_sql.clone()]);
30+
lines.join("\n")
2631
}

‎crates/sqlbuild-analysis/src/planner_identity/models.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ pub struct FunctionFingerprintFields {
5252
pub runtime_version: String,
5353
pub entry_point: String,
5454
pub packages: Vec<String>,
55+
/// A Python table function's frame library; other functions have none.
56+
pub dataframe: Option<String>,
5557
pub body_sql: String,
5658
}
5759

‎crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs‎

Lines changed: 36 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -232,24 +232,43 @@ fn given_deep_chain_when_walking_graph_then_does_not_overflow_the_stack() {
232232

233233
#[test]
234234
fn given_function_fields_when_rendering_then_writes_one_field_per_line() {
235-
let test_cases = [FunctionSqlTestCase {
236-
description: "python function with packages and an unnamed argument",
237-
fields: FunctionFingerprintFields {
238-
language: "python".to_owned(),
239-
arguments: vec![
240-
("x".to_owned(), "int".to_owned()),
241-
(String::new(), "text".to_owned()),
242-
],
243-
returns: "table".to_owned(),
244-
return_columns: vec![("total".to_owned(), "double".to_owned())],
245-
runtime_version: "3.12".to_owned(),
246-
entry_point: String::new(),
247-
packages: texts(&["numpy", "pandas"]),
248-
body_sql: "return x\r\n".to_owned(),
235+
let test_cases = [
236+
FunctionSqlTestCase {
237+
description: "python function with packages and an unnamed argument",
238+
fields: FunctionFingerprintFields {
239+
language: "python".to_owned(),
240+
arguments: vec![
241+
("x".to_owned(), "int".to_owned()),
242+
(String::new(), "text".to_owned()),
243+
],
244+
returns: "table".to_owned(),
245+
return_columns: vec![("total".to_owned(), "double".to_owned())],
246+
runtime_version: "3.12".to_owned(),
247+
entry_point: String::new(),
248+
packages: texts(&["numpy", "pandas"]),
249+
dataframe: None,
250+
body_sql: "return x\r\n".to_owned(),
251+
},
252+
expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\
253+
runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n",
249254
},
250-
expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\
251-
runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n",
252-
}];
255+
FunctionSqlTestCase {
256+
description: "a python table function's frame library changes its definition",
257+
fields: FunctionFingerprintFields {
258+
language: "python".to_owned(),
259+
arguments: vec![("x".to_owned(), "int".to_owned())],
260+
returns: "TABLE".to_owned(),
261+
return_columns: vec![("total".to_owned(), "double".to_owned())],
262+
runtime_version: "3.12".to_owned(),
263+
entry_point: "main".to_owned(),
264+
packages: Vec::new(),
265+
dataframe: Some("pandas".to_owned()),
266+
body_sql: "return x".to_owned(),
267+
},
268+
expected_sql: "language=python\narguments=x:int\nreturns=TABLE\nreturn_columns=total:double\n\
269+
runtime_version=3.12\nentry_point=main\npackages=\ndataframe=pandas\nbody=\nreturn x",
270+
},
271+
];
253272
for test_case in test_cases {
254273
assert_eq!(
255274
function_fingerprint_sql(&test_case.fields),

‎crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use crate::relation_markers::main::lateral_start::lateral_start;
44
use crate::semantic_checks::constants::UDF_NAME_PREFIX;
55
use regex::Regex;
66
use sqlbuild_sqltext::compiler::main::placeholder_default_edits::placeholder_default_edits;
7+
use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open;
78
use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax;
89
use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
910
use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren;
@@ -146,8 +147,8 @@ pub(super) fn normalize(
146147
let mut edits: Vec<Edit> = Vec::new();
147148
let mut cursor: CharCursor = CharCursor::default();
148149
let mut consumed_until = 0;
149-
let non_code: Vec<(usize, usize)> =
150-
dialect_non_code_ranges(&result.sql, &dialect_lexical_syntax(dialect));
150+
let syntax = dialect_lexical_syntax(dialect);
151+
let non_code: Vec<(usize, usize)> = dialect_non_code_ranges(&result.sql, &syntax);
151152
for captures in FUNCTIONS
152153
.as_ref()
153154
.map_err(Clone::clone)?
@@ -169,6 +170,11 @@ pub(super) fn normalize(
169170
end = matching_paren(result.sql.as_bytes(), whole.end(), QuotePolicy::COMPILER)
170171
.map_err(|error| format!("invalid table-function analysis input: {error:?}"))?
171172
+ 1;
173+
if let Some(open) = over_clause_open(&result.sql, end, &syntax.line_comment_prefixes) {
174+
end = matching_paren(result.sql.as_bytes(), open, QuotePolicy::COMPILER).map_err(
175+
|error| format!("invalid table-function partition clause: {error:?}"),
176+
)? + 1;
177+
}
172178
start = lateral_start(&result.sql, start, |index| {
173179
!non_code
174180
.iter()

‎crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,18 @@ fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names
204204
sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l",
205205
expected_sql: "SELECT l.a FROM b, -- not lateral\n__sqlbuild_table_function_f AS l",
206206
},
207+
NativeStubTestCase {
208+
dialect: "snowflake",
209+
description: "a whole-partition table function's OVER clause belongs to its stub relation",
210+
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (PARTITION BY COALESCE(b.x, b.id)) AS l",
211+
expected_sql: "SELECT l.a FROM b, __sqlbuild_table_function_f AS l",
212+
},
213+
NativeStubTestCase {
214+
dialect: "snowflake",
215+
description: "an alias starting with over is not an OVER clause",
216+
sql: "SELECT overview.a FROM __table_fn(\"f\")(1) overview",
217+
expected_sql: "SELECT overview.a FROM __sqlbuild_table_function_f overview",
218+
},
207219
];
208220
for test_case in test_cases {
209221
assert_eq!(

‎crates/sqlbuild-attachments/src/functions/_helpers/header_parsing.rs‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ use crate::functions::models::{
1212
/// Python's error messages for one function file.
1313
pub(crate) struct HeaderErrors<'path> {
1414
pub(crate) language: FunctionLanguage,
15+
/// A Python `@table_fn(...)` file, whose `returns` maps each column to its type.
16+
pub(crate) table_function: bool,
1517
pub(crate) relative_path: &'path str,
1618
}
1719

@@ -51,6 +53,7 @@ pub(crate) fn parsed_header(
5153
errors: &HeaderErrors<'_>,
5254
) -> FunctionHeader {
5355
let python: bool = errors.language == FunctionLanguage::Python;
56+
let python_table: bool = python && errors.table_function;
5457
let mut parsed = FunctionHeader {
5558
arguments: Vec::new(),
5659
returns: None,
@@ -63,7 +66,7 @@ pub(crate) fn parsed_header(
6366
};
6467
let raw_returns: Option<&HeaderValue> = lookup(header, "returns");
6568
let scalar_returns: Option<String> = stripped_text(raw_returns);
66-
if raw_returns.is_none() || (python && scalar_returns.is_none()) {
69+
if raw_returns.is_none() || (python && !python_table && scalar_returns.is_none()) {
6770
return failed(
6871
parsed,
6972
errors.fail(HeaderStage::Start, "must declare returns"),
@@ -84,7 +87,13 @@ pub(crate) fn parsed_header(
8487
return failed(parsed, failure);
8588
}
8689
}
87-
if let Some(text) = scalar_returns {
90+
if python_table {
91+
let (columns, failure) = python_table_returns(raw_returns, errors);
92+
parsed.returns = Some(FunctionReturns::Table(columns));
93+
if let Some(failure) = failure {
94+
return failed(parsed, failure);
95+
}
96+
} else if let Some(text) = scalar_returns {
8897
parsed.returns = Some(FunctionReturns::Type(text));
8998
} else {
9099
let (columns, failure) = table_returns(raw_returns, errors);
@@ -136,6 +145,28 @@ fn failed(parsed: FunctionHeader, failure: HeaderFailure) -> FunctionHeader {
136145
}
137146
}
138147

148+
/// A `@table_fn(returns={...})` column map, which must name at least one column.
149+
fn python_table_returns(
150+
raw_returns: Option<&HeaderValue>,
151+
errors: &HeaderErrors<'_>,
152+
) -> (Vec<NamedType>, Option<HeaderFailure>) {
153+
let shape_error = || {
154+
errors.fail(
155+
HeaderStage::Returns,
156+
"returns must map each column name to its type",
157+
)
158+
};
159+
match raw_returns {
160+
Some(HeaderValue::Map(entries)) if !entries.is_empty() => {
161+
let (columns, error) = named_types(entries);
162+
let failure =
163+
error.map(|error| errors.named_type(HeaderStage::Returns, error, "return column"));
164+
(columns, failure)
165+
}
166+
_ => (Vec::new(), Some(shape_error())),
167+
}
168+
}
169+
139170
/// Python's `returns (table (...))` columns: exactly the `table` key with a non-empty column map.
140171
fn table_returns(
141172
raw_returns: Option<&HeaderValue>,

0 commit comments

Comments
 (0)