Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,15 @@ use regex::{Captures, Regex};

use crate::compiler::_helpers::sql_tests::planning::compile_error;
use crate::relation_markers::main::lateral_start::lateral_start;
use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open;
use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren;
use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace;
use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed};

pub(crate) fn replace_callable_markers<F>(
sql: &str,
(pattern, drop_lateral): (&Regex, bool),
(pattern, drop_lateral, table_call): (&Regex, bool, bool),
syntax: &LexicalSyntax,
mut replacement: F,
) -> Result<(String, HashSet<String>), String>
Expand All @@ -35,7 +36,13 @@ where
if !sql[suffix_start..].starts_with('(') {
continue;
}
let suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
let mut suffix_end = matching_paren_end(sql, suffix_start, syntax)?;
if table_call
&& let Some(open) = over_clause_open(sql, suffix_end, &syntax.line_comment_prefixes)
&& !protected.contains(open)
{
suffix_end = matching_paren_end(sql, open, syntax)?;
}
let Some(name) = captures.get(1).map(|value| value.as_str()) else {
continue;
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1709,7 +1709,7 @@ fn resolve_table_function_fixtures<'a>(
}
let (replaced, reached) = replace_callable_markers(
sql,
(&patterns.table_function, true),
(&patterns.table_function, true, true),
&patterns.lexical,
|name, _call_suffix| {
fixtures
Expand Down Expand Up @@ -1750,7 +1750,7 @@ fn resolve_function_calls(
};
let (result, _) = replace_callable_markers(
sql,
(pattern, false),
(pattern, false, table_function),
&patterns.lexical,
|name, call_suffix| {
let function = functions.get(name)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,31 +55,57 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la
description: "a mock after a comma LATERAL stands alone",
sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l",
drop_lateral: true,
expected_sql: "SELECT l.a FROM b, mock_f AS l",
table_call: true,
expected_sql: "SELECT l.a FROM b, mock_f[(b.id)] AS l",
},
CallableMarkerTestCase {
description: "a mock after JOIN lateral, in any case, stands alone",
sql: "SELECT l.a FROM b JOIN lateral __table_fn(\"f\")(b.id) AS l ON TRUE",
drop_lateral: true,
expected_sql: "SELECT l.a FROM b JOIN mock_f AS l ON TRUE",
table_call: true,
expected_sql: "SELECT l.a FROM b JOIN mock_f[(b.id)] AS l ON TRUE",
},
CallableMarkerTestCase {
description: "a rendered call keeps its LATERAL",
sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l",
drop_lateral: false,
expected_sql: "SELECT l.a FROM b, LATERAL mock_f AS l",
table_call: true,
expected_sql: "SELECT l.a FROM b, LATERAL mock_f[(b.id)] AS l",
},
CallableMarkerTestCase {
description: "lateral ending a comment before the call is kept",
sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l",
drop_lateral: true,
expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f AS l",
table_call: true,
expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f[(1)] AS l",
},
CallableMarkerTestCase {
description: "a word ending in lateral before the call is kept",
sql: "SELECT l.a FROM collateral, __table_fn(\"f\")(1) AS l",
drop_lateral: true,
expected_sql: "SELECT l.a FROM collateral, mock_f AS l",
table_call: true,
expected_sql: "SELECT l.a FROM collateral, mock_f[(1)] AS l",
},
CallableMarkerTestCase {
description: "a table function call's suffix takes its OVER clause with it",
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id) OVER (PARTITION BY (b.x)) AS l",
drop_lateral: true,
table_call: true,
expected_sql: "SELECT l.a FROM b, mock_f[(b.id) OVER (PARTITION BY (b.x))] AS l",
},
CallableMarkerTestCase {
description: "an OVER clause in any case and across lines belongs to the call",
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (partition by b.x) l",
drop_lateral: false,
table_call: true,
expected_sql: "SELECT l.a FROM b, mock_f[(b.id)\n over (partition by b.x)] l",
},
CallableMarkerTestCase {
description: "a scalar call keeps a window clause after it",
sql: "SELECT __table_fn(\"f\")(b.id) OVER (ORDER BY b.x) FROM b",
drop_lateral: false,
table_call: false,
expected_sql: "SELECT mock_f[(b.id)] OVER (ORDER BY b.x) FROM b",
},
];
let pattern = Regex::new(r#"(?i)__table_fn\("([^"]+)"\)"#).expect("valid marker pattern");
Expand All @@ -90,9 +116,9 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la
for test_case in test_cases {
let replaced = replace_callable_markers(
test_case.sql,
(&pattern, test_case.drop_lateral),
(&pattern, test_case.drop_lateral, test_case.table_call),
&syntax,
|name, _| Some((format!("mock_{name}"), true)),
|name, suffix| Some((format!("mock_{name}[{suffix}]"), true)),
)
.map(|(sql, _)| sql);
assert_eq!(
Expand Down
3 changes: 2 additions & 1 deletion crates/sqlbuild-analysis/src/compiler/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,12 @@ pub(crate) struct MarkerNamesTestCase {
pub(crate) expected_replaced_sql: &'static str,
}

/// A table function call replaced by a name, with or without dropping a `LATERAL` before it.
/// A call replaced by a name or a rendered call, with or without its `LATERAL` and `OVER (...)`.
pub(crate) struct CallableMarkerTestCase {
pub(crate) description: &'static str,
pub(crate) sql: &'static str,
pub(crate) drop_lateral: bool,
pub(crate) table_call: bool,
pub(crate) expected_sql: &'static str,
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use crate::planner_identity::models::FunctionFingerprintFields;

/// The language, signature, runtime fields and body, one field per line.
/// The language, signature, runtime fields, any frame library and body, one field per line.
pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String {
let render = |pairs: &[(String, String)]| -> String {
pairs
Expand All @@ -11,16 +11,21 @@ pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String {
.collect::<Vec<String>>()
.join(",")
};
[
let mut lines: Vec<String> = vec![
format!("language={}", fields.language),
format!("arguments={}", render(&fields.arguments)),
format!("returns={}", fields.returns),
format!("return_columns={}", render(&fields.return_columns)),
format!("runtime_version={}", fields.runtime_version),
format!("entry_point={}", fields.entry_point),
format!("packages={}", fields.packages.join(",")),
"body=".to_owned(),
fields.body_sql.clone(),
]
.join("\n")
];
lines.extend(
fields
.dataframe
.as_ref()
.map(|dataframe| format!("dataframe={dataframe}")),
);
lines.extend(["body=".to_owned(), fields.body_sql.clone()]);
lines.join("\n")
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-analysis/src/planner_identity/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ pub struct FunctionFingerprintFields {
pub runtime_version: String,
pub entry_point: String,
pub packages: Vec<String>,
/// A Python table function's frame library; other functions have none.
pub dataframe: Option<String>,
pub body_sql: String,
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,24 +232,43 @@ fn given_deep_chain_when_walking_graph_then_does_not_overflow_the_stack() {

#[test]
fn given_function_fields_when_rendering_then_writes_one_field_per_line() {
let test_cases = [FunctionSqlTestCase {
description: "python function with packages and an unnamed argument",
fields: FunctionFingerprintFields {
language: "python".to_owned(),
arguments: vec![
("x".to_owned(), "int".to_owned()),
(String::new(), "text".to_owned()),
],
returns: "table".to_owned(),
return_columns: vec![("total".to_owned(), "double".to_owned())],
runtime_version: "3.12".to_owned(),
entry_point: String::new(),
packages: texts(&["numpy", "pandas"]),
body_sql: "return x\r\n".to_owned(),
let test_cases = [
FunctionSqlTestCase {
description: "python function with packages and an unnamed argument",
fields: FunctionFingerprintFields {
language: "python".to_owned(),
arguments: vec![
("x".to_owned(), "int".to_owned()),
(String::new(), "text".to_owned()),
],
returns: "table".to_owned(),
return_columns: vec![("total".to_owned(), "double".to_owned())],
runtime_version: "3.12".to_owned(),
entry_point: String::new(),
packages: texts(&["numpy", "pandas"]),
dataframe: None,
body_sql: "return x\r\n".to_owned(),
},
expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\
runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n",
},
expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\
runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n",
}];
FunctionSqlTestCase {
description: "a python table function's frame library changes its definition",
fields: FunctionFingerprintFields {
language: "python".to_owned(),
arguments: vec![("x".to_owned(), "int".to_owned())],
returns: "TABLE".to_owned(),
return_columns: vec![("total".to_owned(), "double".to_owned())],
runtime_version: "3.12".to_owned(),
entry_point: "main".to_owned(),
packages: Vec::new(),
dataframe: Some("pandas".to_owned()),
body_sql: "return x".to_owned(),
},
expected_sql: "language=python\narguments=x:int\nreturns=TABLE\nreturn_columns=total:double\n\
runtime_version=3.12\nentry_point=main\npackages=\ndataframe=pandas\nbody=\nreturn x",
},
];
for test_case in test_cases {
assert_eq!(
function_fingerprint_sql(&test_case.fields),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use crate::relation_markers::main::lateral_start::lateral_start;
use crate::semantic_checks::constants::UDF_NAME_PREFIX;
use regex::Regex;
use sqlbuild_sqltext::compiler::main::placeholder_default_edits::placeholder_default_edits;
use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open;
use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax;
use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges;
use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren;
Expand Down Expand Up @@ -146,8 +147,8 @@ pub(super) fn normalize(
let mut edits: Vec<Edit> = Vec::new();
let mut cursor: CharCursor = CharCursor::default();
let mut consumed_until = 0;
let non_code: Vec<(usize, usize)> =
dialect_non_code_ranges(&result.sql, &dialect_lexical_syntax(dialect));
let syntax = dialect_lexical_syntax(dialect);
let non_code: Vec<(usize, usize)> = dialect_non_code_ranges(&result.sql, &syntax);
for captures in FUNCTIONS
.as_ref()
.map_err(Clone::clone)?
Expand All @@ -169,6 +170,11 @@ pub(super) fn normalize(
end = matching_paren(result.sql.as_bytes(), whole.end(), QuotePolicy::COMPILER)
.map_err(|error| format!("invalid table-function analysis input: {error:?}"))?
+ 1;
if let Some(open) = over_clause_open(&result.sql, end, &syntax.line_comment_prefixes) {
end = matching_paren(result.sql.as_bytes(), open, QuotePolicy::COMPILER).map_err(
|error| format!("invalid table-function partition clause: {error:?}"),
)? + 1;
}
start = lateral_start(&result.sql, start, |index| {
!non_code
.iter()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,18 @@ fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names
sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l",
expected_sql: "SELECT l.a FROM b, -- not lateral\n__sqlbuild_table_function_f AS l",
},
NativeStubTestCase {
dialect: "snowflake",
description: "a whole-partition table function's OVER clause belongs to its stub relation",
sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (PARTITION BY COALESCE(b.x, b.id)) AS l",
expected_sql: "SELECT l.a FROM b, __sqlbuild_table_function_f AS l",
},
NativeStubTestCase {
dialect: "snowflake",
description: "an alias starting with over is not an OVER clause",
sql: "SELECT overview.a FROM __table_fn(\"f\")(1) overview",
expected_sql: "SELECT overview.a FROM __sqlbuild_table_function_f overview",
},
];
for test_case in test_cases {
assert_eq!(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ use crate::functions::models::{
/// Python's error messages for one function file.
pub(crate) struct HeaderErrors<'path> {
pub(crate) language: FunctionLanguage,
/// A Python `@table_fn(...)` file, whose `returns` maps each column to its type.
pub(crate) table_function: bool,
pub(crate) relative_path: &'path str,
}

Expand Down Expand Up @@ -51,6 +53,7 @@ pub(crate) fn parsed_header(
errors: &HeaderErrors<'_>,
) -> FunctionHeader {
let python: bool = errors.language == FunctionLanguage::Python;
let python_table: bool = python && errors.table_function;
let mut parsed = FunctionHeader {
arguments: Vec::new(),
returns: None,
Expand All @@ -63,7 +66,7 @@ pub(crate) fn parsed_header(
};
let raw_returns: Option<&HeaderValue> = lookup(header, "returns");
let scalar_returns: Option<String> = stripped_text(raw_returns);
if raw_returns.is_none() || (python && scalar_returns.is_none()) {
if raw_returns.is_none() || (python && !python_table && scalar_returns.is_none()) {
return failed(
parsed,
errors.fail(HeaderStage::Start, "must declare returns"),
Expand All @@ -84,7 +87,13 @@ pub(crate) fn parsed_header(
return failed(parsed, failure);
}
}
if let Some(text) = scalar_returns {
if python_table {
let (columns, failure) = python_table_returns(raw_returns, errors);
parsed.returns = Some(FunctionReturns::Table(columns));
if let Some(failure) = failure {
return failed(parsed, failure);
}
} else if let Some(text) = scalar_returns {
parsed.returns = Some(FunctionReturns::Type(text));
} else {
let (columns, failure) = table_returns(raw_returns, errors);
Expand Down Expand Up @@ -136,6 +145,28 @@ fn failed(parsed: FunctionHeader, failure: HeaderFailure) -> FunctionHeader {
}
}

/// A `@table_fn(returns={...})` column map, which must name at least one column.
fn python_table_returns(
raw_returns: Option<&HeaderValue>,
errors: &HeaderErrors<'_>,
) -> (Vec<NamedType>, Option<HeaderFailure>) {
let shape_error = || {
errors.fail(
HeaderStage::Returns,
"returns must map each column name to its type",
)
};
match raw_returns {
Some(HeaderValue::Map(entries)) if !entries.is_empty() => {
let (columns, error) = named_types(entries);
let failure =
error.map(|error| errors.named_type(HeaderStage::Returns, error, "return column"));
(columns, failure)
}
_ => (Vec::new(), Some(shape_error())),
}
}

/// Python's `returns (table (...))` columns: exactly the `table` key with a non-empty column map.
fn table_returns(
raw_returns: Option<&HeaderValue>,
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlbuild-attachments/src/functions/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,7 @@ pub(crate) const SOURCE_KIND: &str = "source";
pub(crate) const DBT_REF_KIND: &str = "dbt_ref";
pub(crate) const UDF_KIND: &str = "udf";
pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn";
/// The frame library whose columns hold Snowflake's fixed-point values as Python objects.
pub(crate) const POLARS_DATAFRAME: &str = "polars";
/// The fixed-point type names Snowflake passes to Python as `Decimal`.
pub(crate) const FIXED_POINT_TYPES: [&str; 4] = ["NUMBER", "DECIMAL", "DEC", "NUMERIC"];
Loading
Loading