diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs index 6e485feec3..bada5187ea 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/markers.rs @@ -6,6 +6,7 @@ use regex::{Captures, Regex}; use crate::compiler::_helpers::sql_tests::planning::compile_error; use crate::relation_markers::main::lateral_start::lateral_start; +use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open; use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_paren; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::skip_whitespace::skip_whitespace; @@ -13,7 +14,7 @@ use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; pub(crate) fn replace_callable_markers( sql: &str, - (pattern, drop_lateral): (&Regex, bool), + (pattern, drop_lateral, table_call): (&Regex, bool, bool), syntax: &LexicalSyntax, mut replacement: F, ) -> Result<(String, HashSet), String> @@ -35,7 +36,13 @@ where if !sql[suffix_start..].starts_with('(') { continue; } - let suffix_end = matching_paren_end(sql, suffix_start, syntax)?; + let mut suffix_end = matching_paren_end(sql, suffix_start, syntax)?; + if table_call + && let Some(open) = over_clause_open(sql, suffix_end, &syntax.line_comment_prefixes) + && !protected.contains(open) + { + suffix_end = matching_paren_end(sql, open, syntax)?; + } let Some(name) = captures.get(1).map(|value| value.as_str()) else { continue; }; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 93913583f3..954566bf1c 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -1709,7 +1709,7 @@ fn resolve_table_function_fixtures<'a>( } let (replaced, reached) = replace_callable_markers( sql, - (&patterns.table_function, true), + (&patterns.table_function, true, true), &patterns.lexical, |name, _call_suffix| { fixtures @@ -1750,7 +1750,7 @@ fn resolve_function_calls( }; let (result, _) = replace_callable_markers( sql, - (pattern, false), + (pattern, false, table_function), &patterns.lexical, |name, call_suffix| { let function = functions.get(name)?; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs index bffdb88a2a..7a010e28c3 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_markers.rs @@ -55,31 +55,57 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la description: "a mock after a comma LATERAL stands alone", sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l", drop_lateral: true, - expected_sql: "SELECT l.a FROM b, mock_f AS l", + table_call: true, + expected_sql: "SELECT l.a FROM b, mock_f[(b.id)] AS l", }, CallableMarkerTestCase { description: "a mock after JOIN lateral, in any case, stands alone", sql: "SELECT l.a FROM b JOIN lateral __table_fn(\"f\")(b.id) AS l ON TRUE", drop_lateral: true, - expected_sql: "SELECT l.a FROM b JOIN mock_f AS l ON TRUE", + table_call: true, + expected_sql: "SELECT l.a FROM b JOIN mock_f[(b.id)] AS l ON TRUE", }, CallableMarkerTestCase { description: "a rendered call keeps its LATERAL", sql: "SELECT l.a FROM b, LATERAL __table_fn(\"f\")(b.id) AS l", drop_lateral: false, - expected_sql: "SELECT l.a FROM b, LATERAL mock_f AS l", + table_call: true, + expected_sql: "SELECT l.a FROM b, LATERAL mock_f[(b.id)] AS l", }, CallableMarkerTestCase { description: "lateral ending a comment before the call is kept", sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l", drop_lateral: true, - expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f AS l", + table_call: true, + expected_sql: "SELECT l.a FROM b, -- not lateral\nmock_f[(1)] AS l", }, CallableMarkerTestCase { description: "a word ending in lateral before the call is kept", sql: "SELECT l.a FROM collateral, __table_fn(\"f\")(1) AS l", drop_lateral: true, - expected_sql: "SELECT l.a FROM collateral, mock_f AS l", + table_call: true, + expected_sql: "SELECT l.a FROM collateral, mock_f[(1)] AS l", + }, + CallableMarkerTestCase { + description: "a table function call's suffix takes its OVER clause with it", + sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id) OVER (PARTITION BY (b.x)) AS l", + drop_lateral: true, + table_call: true, + expected_sql: "SELECT l.a FROM b, mock_f[(b.id) OVER (PARTITION BY (b.x))] AS l", + }, + CallableMarkerTestCase { + description: "an OVER clause in any case and across lines belongs to the call", + sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (partition by b.x) l", + drop_lateral: false, + table_call: true, + expected_sql: "SELECT l.a FROM b, mock_f[(b.id)\n over (partition by b.x)] l", + }, + CallableMarkerTestCase { + description: "a scalar call keeps a window clause after it", + sql: "SELECT __table_fn(\"f\")(b.id) OVER (ORDER BY b.x) FROM b", + drop_lateral: false, + table_call: false, + expected_sql: "SELECT mock_f[(b.id)] OVER (ORDER BY b.x) FROM b", }, ]; let pattern = Regex::new(r#"(?i)__table_fn\("([^"]+)"\)"#).expect("valid marker pattern"); @@ -90,9 +116,9 @@ fn given_a_lateral_table_function_call_when_replacing_it_with_a_relation_then_la for test_case in test_cases { let replaced = replace_callable_markers( test_case.sql, - (&pattern, test_case.drop_lateral), + (&pattern, test_case.drop_lateral, test_case.table_call), &syntax, - |name, _| Some((format!("mock_{name}"), true)), + |name, suffix| Some((format!("mock_{name}[{suffix}]"), true)), ) .map(|(sql, _)| sql); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 67a3a43ecc..b0404a61b6 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -63,11 +63,12 @@ pub(crate) struct MarkerNamesTestCase { pub(crate) expected_replaced_sql: &'static str, } -/// A table function call replaced by a name, with or without dropping a `LATERAL` before it. +/// A call replaced by a name or a rendered call, with or without its `LATERAL` and `OVER (...)`. pub(crate) struct CallableMarkerTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, pub(crate) drop_lateral: bool, + pub(crate) table_call: bool, pub(crate) expected_sql: &'static str, } diff --git a/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs index 8317a9fbf2..4bd6489af1 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/main/function_fingerprint_sql.rs @@ -2,7 +2,7 @@ use crate::planner_identity::models::FunctionFingerprintFields; -/// The language, signature, runtime fields and body, one field per line. +/// The language, signature, runtime fields, any frame library and body, one field per line. pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String { let render = |pairs: &[(String, String)]| -> String { pairs @@ -11,7 +11,7 @@ pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String { .collect::>() .join(",") }; - [ + let mut lines: Vec = vec![ format!("language={}", fields.language), format!("arguments={}", render(&fields.arguments)), format!("returns={}", fields.returns), @@ -19,8 +19,13 @@ pub fn function_fingerprint_sql(fields: &FunctionFingerprintFields) -> String { format!("runtime_version={}", fields.runtime_version), format!("entry_point={}", fields.entry_point), format!("packages={}", fields.packages.join(",")), - "body=".to_owned(), - fields.body_sql.clone(), - ] - .join("\n") + ]; + lines.extend( + fields + .dataframe + .as_ref() + .map(|dataframe| format!("dataframe={dataframe}")), + ); + lines.extend(["body=".to_owned(), fields.body_sql.clone()]); + lines.join("\n") } diff --git a/crates/sqlbuild-analysis/src/planner_identity/models.rs b/crates/sqlbuild-analysis/src/planner_identity/models.rs index 0388d75735..24db69ebc7 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/models.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/models.rs @@ -52,6 +52,8 @@ pub struct FunctionFingerprintFields { pub runtime_version: String, pub entry_point: String, pub packages: Vec, + /// A Python table function's frame library; other functions have none. + pub dataframe: Option, pub body_sql: String, } diff --git a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs index da55a9535b..bbe473046e 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/tests/test_planner_identity.rs @@ -232,24 +232,43 @@ fn given_deep_chain_when_walking_graph_then_does_not_overflow_the_stack() { #[test] fn given_function_fields_when_rendering_then_writes_one_field_per_line() { - let test_cases = [FunctionSqlTestCase { - description: "python function with packages and an unnamed argument", - fields: FunctionFingerprintFields { - language: "python".to_owned(), - arguments: vec![ - ("x".to_owned(), "int".to_owned()), - (String::new(), "text".to_owned()), - ], - returns: "table".to_owned(), - return_columns: vec![("total".to_owned(), "double".to_owned())], - runtime_version: "3.12".to_owned(), - entry_point: String::new(), - packages: texts(&["numpy", "pandas"]), - body_sql: "return x\r\n".to_owned(), + let test_cases = [ + FunctionSqlTestCase { + description: "python function with packages and an unnamed argument", + fields: FunctionFingerprintFields { + language: "python".to_owned(), + arguments: vec![ + ("x".to_owned(), "int".to_owned()), + (String::new(), "text".to_owned()), + ], + returns: "table".to_owned(), + return_columns: vec![("total".to_owned(), "double".to_owned())], + runtime_version: "3.12".to_owned(), + entry_point: String::new(), + packages: texts(&["numpy", "pandas"]), + dataframe: None, + body_sql: "return x\r\n".to_owned(), + }, + expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\ + runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n", }, - expected_sql: "language=python\narguments=x:int,:text\nreturns=table\nreturn_columns=total:double\n\ - runtime_version=3.12\nentry_point=\npackages=numpy,pandas\nbody=\nreturn x\r\n", - }]; + FunctionSqlTestCase { + description: "a python table function's frame library changes its definition", + fields: FunctionFingerprintFields { + language: "python".to_owned(), + arguments: vec![("x".to_owned(), "int".to_owned())], + returns: "TABLE".to_owned(), + return_columns: vec![("total".to_owned(), "double".to_owned())], + runtime_version: "3.12".to_owned(), + entry_point: "main".to_owned(), + packages: Vec::new(), + dataframe: Some("pandas".to_owned()), + body_sql: "return x".to_owned(), + }, + expected_sql: "language=python\narguments=x:int\nreturns=TABLE\nreturn_columns=total:double\n\ + runtime_version=3.12\nentry_point=main\npackages=\ndataframe=pandas\nbody=\nreturn x", + }, + ]; for test_case in test_cases { assert_eq!( function_fingerprint_sql(&test_case.fields), diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 814068823b..9689e7d970 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -4,6 +4,7 @@ use crate::relation_markers::main::lateral_start::lateral_start; use crate::semantic_checks::constants::UDF_NAME_PREFIX; use regex::Regex; use sqlbuild_sqltext::compiler::main::placeholder_default_edits::placeholder_default_edits; +use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open; use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; @@ -146,8 +147,8 @@ pub(super) fn normalize( let mut edits: Vec = Vec::new(); let mut cursor: CharCursor = CharCursor::default(); let mut consumed_until = 0; - let non_code: Vec<(usize, usize)> = - dialect_non_code_ranges(&result.sql, &dialect_lexical_syntax(dialect)); + let syntax = dialect_lexical_syntax(dialect); + let non_code: Vec<(usize, usize)> = dialect_non_code_ranges(&result.sql, &syntax); for captures in FUNCTIONS .as_ref() .map_err(Clone::clone)? @@ -169,6 +170,11 @@ pub(super) fn normalize( end = matching_paren(result.sql.as_bytes(), whole.end(), QuotePolicy::COMPILER) .map_err(|error| format!("invalid table-function analysis input: {error:?}"))? + 1; + if let Some(open) = over_clause_open(&result.sql, end, &syntax.line_comment_prefixes) { + end = matching_paren(result.sql.as_bytes(), open, QuotePolicy::COMPILER).map_err( + |error| format!("invalid table-function partition clause: {error:?}"), + )? + 1; + } start = lateral_start(&result.sql, start, |index| { !non_code .iter() diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index f6f101771c..efe2c4d5c9 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -204,6 +204,18 @@ fn given_cursor_stubs_when_normalizing_for_a_native_parser_then_only_whole_names sql: "SELECT l.a FROM b, -- not lateral\n__table_fn(\"f\")(1) AS l", expected_sql: "SELECT l.a FROM b, -- not lateral\n__sqlbuild_table_function_f AS l", }, + NativeStubTestCase { + dialect: "snowflake", + description: "a whole-partition table function's OVER clause belongs to its stub relation", + sql: "SELECT l.a FROM b, __table_fn(\"f\")(b.id)\n over (PARTITION BY COALESCE(b.x, b.id)) AS l", + expected_sql: "SELECT l.a FROM b, __sqlbuild_table_function_f AS l", + }, + NativeStubTestCase { + dialect: "snowflake", + description: "an alias starting with over is not an OVER clause", + sql: "SELECT overview.a FROM __table_fn(\"f\")(1) overview", + expected_sql: "SELECT overview.a FROM __sqlbuild_table_function_f overview", + }, ]; for test_case in test_cases { assert_eq!( diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/header_parsing.rs b/crates/sqlbuild-attachments/src/functions/_helpers/header_parsing.rs index 834d5ca2de..f7ca76c029 100644 --- a/crates/sqlbuild-attachments/src/functions/_helpers/header_parsing.rs +++ b/crates/sqlbuild-attachments/src/functions/_helpers/header_parsing.rs @@ -12,6 +12,8 @@ use crate::functions::models::{ /// Python's error messages for one function file. pub(crate) struct HeaderErrors<'path> { pub(crate) language: FunctionLanguage, + /// A Python `@table_fn(...)` file, whose `returns` maps each column to its type. + pub(crate) table_function: bool, pub(crate) relative_path: &'path str, } @@ -51,6 +53,7 @@ pub(crate) fn parsed_header( errors: &HeaderErrors<'_>, ) -> FunctionHeader { let python: bool = errors.language == FunctionLanguage::Python; + let python_table: bool = python && errors.table_function; let mut parsed = FunctionHeader { arguments: Vec::new(), returns: None, @@ -63,7 +66,7 @@ pub(crate) fn parsed_header( }; let raw_returns: Option<&HeaderValue> = lookup(header, "returns"); let scalar_returns: Option = stripped_text(raw_returns); - if raw_returns.is_none() || (python && scalar_returns.is_none()) { + if raw_returns.is_none() || (python && !python_table && scalar_returns.is_none()) { return failed( parsed, errors.fail(HeaderStage::Start, "must declare returns"), @@ -84,7 +87,13 @@ pub(crate) fn parsed_header( return failed(parsed, failure); } } - if let Some(text) = scalar_returns { + if python_table { + let (columns, failure) = python_table_returns(raw_returns, errors); + parsed.returns = Some(FunctionReturns::Table(columns)); + if let Some(failure) = failure { + return failed(parsed, failure); + } + } else if let Some(text) = scalar_returns { parsed.returns = Some(FunctionReturns::Type(text)); } else { let (columns, failure) = table_returns(raw_returns, errors); @@ -136,6 +145,28 @@ fn failed(parsed: FunctionHeader, failure: HeaderFailure) -> FunctionHeader { } } +/// A `@table_fn(returns={...})` column map, which must name at least one column. +fn python_table_returns( + raw_returns: Option<&HeaderValue>, + errors: &HeaderErrors<'_>, +) -> (Vec, Option) { + let shape_error = || { + errors.fail( + HeaderStage::Returns, + "returns must map each column name to its type", + ) + }; + match raw_returns { + Some(HeaderValue::Map(entries)) if !entries.is_empty() => { + let (columns, error) = named_types(entries); + let failure = + error.map(|error| errors.named_type(HeaderStage::Returns, error, "return column")); + (columns, failure) + } + _ => (Vec::new(), Some(shape_error())), + } +} + /// Python's `returns (table (...))` columns: exactly the `table` key with a non-empty column map. fn table_returns( raw_returns: Option<&HeaderValue>, diff --git a/crates/sqlbuild-attachments/src/functions/constants.rs b/crates/sqlbuild-attachments/src/functions/constants.rs index f66712c87e..2aa5a4f9cc 100644 --- a/crates/sqlbuild-attachments/src/functions/constants.rs +++ b/crates/sqlbuild-attachments/src/functions/constants.rs @@ -6,3 +6,7 @@ pub(crate) const SOURCE_KIND: &str = "source"; pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; pub(crate) const UDF_KIND: &str = "udf"; pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +/// The frame library whose columns hold Snowflake's fixed-point values as Python objects. +pub(crate) const POLARS_DATAFRAME: &str = "polars"; +/// The fixed-point type names Snowflake passes to Python as `Decimal`. +pub(crate) const FIXED_POINT_TYPES: [&str; 4] = ["NUMBER", "DECIMAL", "DEC", "NUMERIC"]; diff --git a/crates/sqlbuild-attachments/src/functions/main/frame_argument_error.rs b/crates/sqlbuild-attachments/src/functions/main/frame_argument_error.rs new file mode 100644 index 0000000000..398962c167 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/frame_argument_error.rs @@ -0,0 +1,48 @@ +//! A polars table function argument whose type Snowflake passes as Python `Decimal` objects. + +use crate::functions::constants::{FIXED_POINT_TYPES, POLARS_DATAFRAME}; + +/// The first argument of a polars frame whose type is a fixed-point number with a scale above zero. +#[must_use] +pub fn frame_argument_error( + label: &str, + dataframe: Option<&str>, + arguments: &[(String, String)], +) -> Option { + if dataframe != Some(POLARS_DATAFRAME) { + return None; + } + arguments + .iter() + .find(|(_, type_text)| fractional_fixed_point(type_text)) + .map(|(name, type_text)| { + format!( + "{label} argument '{name}' is {type_text}, which Snowflake passes to a polars \ + frame as Python Decimal objects, a slow object column; declare it as FLOAT, or \ + as INTEGER when it has no fraction" + ) + }) +} + +/// Whether `type_text` is `NUMBER`, `DECIMAL`, `DEC` or `NUMERIC` `(p, s)` with `s` above zero. +fn fractional_fixed_point(type_text: &str) -> bool { + let Some((name, parameters)) = type_text.split_once('(') else { + return false; + }; + if !FIXED_POINT_TYPES.contains(&name.trim().to_ascii_uppercase().as_str()) { + return false; + } + parameters + .trim_end() + .trim_end_matches(')') + .split(',') + .nth(1) + .is_some_and(|scale| positive_integer(scale.trim())) +} + +/// Whether `text` is a run of ASCII digits that is not all zeros. +fn positive_integer(text: &str) -> bool { + !text.is_empty() + && text.bytes().all(|byte| byte.is_ascii_digit()) + && text.bytes().any(|byte| byte != b'0') +} diff --git a/crates/sqlbuild-attachments/src/functions/main/mod.rs b/crates/sqlbuild-attachments/src/functions/main/mod.rs index 61b48206b1..fa7e938475 100644 --- a/crates/sqlbuild-attachments/src/functions/main/mod.rs +++ b/crates/sqlbuild-attachments/src/functions/main/mod.rs @@ -1,4 +1,6 @@ +pub mod frame_argument_error; pub mod function_reference_error; pub mod parse_function_header; pub mod resolve_function_namespace; pub mod table_function_arity_error; +pub mod table_function_partition_error; diff --git a/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs b/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs index a22cdfddfd..21e68be4d4 100644 --- a/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs +++ b/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs @@ -3,17 +3,19 @@ use crate::functions::_helpers::header_parsing::{HeaderErrors, parsed_header}; use crate::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; -/// The parsed header of the function file at `relative_path`, with Python's first error. +/// The parsed header of the function file at `relative_path`; `table_function` marks `@table_fn`. #[must_use] pub fn parse_function_header( header: &[(String, HeaderValue)], language: FunctionLanguage, + table_function: bool, relative_path: &str, ) -> FunctionHeader { parsed_header( header, &HeaderErrors { language, + table_function, relative_path, }, ) diff --git a/crates/sqlbuild-attachments/src/functions/main/table_function_partition_error.rs b/crates/sqlbuild-attachments/src/functions/main/table_function_partition_error.rs new file mode 100644 index 0000000000..4eddf65407 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/table_function_partition_error.rs @@ -0,0 +1,23 @@ +//! A whole-partition table function called without `OVER (PARTITION BY ...)`. + +use std::collections::HashSet; + +use crate::functions::constants::TABLE_FUNCTION_KIND; + +/// The first `(kind, name, partitioned)` call of a whole-partition function that has no partition. +#[must_use] +pub fn table_function_partition_error( + owner: &str, + calls: &[(&str, &str, bool)], + whole_partition: &HashSet, +) -> Option { + calls.iter().find_map(|&(kind, name, partitioned)| { + (kind == TABLE_FUNCTION_KIND && !partitioned && whole_partition.contains(name)).then(|| { + format!( + "{owner} calls Python table function '{name}' without OVER (PARTITION BY ...); \ + it receives one whole partition per call, so add OVER (PARTITION BY ) \ + after the call" + ) + }) + }) +} diff --git a/crates/sqlbuild-attachments/src/functions/tests.rs b/crates/sqlbuild-attachments/src/functions/tests.rs index 785e40079f..ee39db821e 100644 --- a/crates/sqlbuild-attachments/src/functions/tests.rs +++ b/crates/sqlbuild-attachments/src/functions/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_frame_argument_error.rs"] +mod frame_argument_error; #[path = "tests/test_function_reference_error.rs"] mod function_reference_error; #[path = "tests/helpers.rs"] @@ -8,5 +10,7 @@ mod parse_function_header; mod resolve_function_namespace; #[path = "tests/test_table_function_arity_error.rs"] mod table_function_arity_error; +#[path = "tests/test_table_function_partition_error.rs"] +mod table_function_partition_error; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_frame_argument_error.rs b/crates/sqlbuild-attachments/src/functions/tests/test_frame_argument_error.rs new file mode 100644 index 0000000000..5ecda76ac3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_frame_argument_error.rs @@ -0,0 +1,83 @@ +use crate::functions::main::frame_argument_error::frame_argument_error; +use crate::functions::tests::test_types::FrameArgumentTestCase; + +const DECIMAL_ERROR: &str = "Python function functions/python/smooth.py argument 'amount' is \ + NUMBER(10, 2), which Snowflake passes to a polars frame as Python Decimal objects, a slow \ + object column; declare it as FLOAT, or as INTEGER when it has no fraction"; + +#[test] +fn given_table_function_arguments_when_checking_the_frame_then_fractional_numbers_fail_for_polars() +{ + let test_cases = [ + FrameArgumentTestCase { + description: "a fractional NUMBER fails for polars", + dataframe: Some("polars"), + argument_type: "NUMBER(10, 2)", + expected_error: Some(DECIMAL_ERROR), + }, + FrameArgumentTestCase { + description: "DECIMAL and NUMERIC are the same type in any case", + dataframe: Some("polars"), + argument_type: "decimal(12,4)", + expected_error: Some( + "Python function functions/python/smooth.py argument 'amount' is \ + decimal(12,4), which Snowflake passes to a polars frame as Python Decimal \ + objects, a slow object column; declare it as FLOAT, or as INTEGER when it has \ + no fraction", + ), + }, + FrameArgumentTestCase { + description: "DEC is a synonym of NUMBER", + dataframe: Some("polars"), + argument_type: "DEC(10, 2)", + expected_error: Some( + "Python function functions/python/smooth.py argument 'amount' is \ + DEC(10, 2), which Snowflake passes to a polars frame as Python Decimal \ + objects, a slow object column; declare it as FLOAT, or as INTEGER when it has \ + no fraction", + ), + }, + FrameArgumentTestCase { + description: "a scale of zero is an integer", + dataframe: Some("polars"), + argument_type: "NUMBER(38, 0)", + expected_error: None, + }, + FrameArgumentTestCase { + description: "NUMBER without a scale defaults to zero", + dataframe: Some("polars"), + argument_type: "NUMBER(38)", + expected_error: None, + }, + FrameArgumentTestCase { + description: "a pandas frame keeps its Decimal objects", + dataframe: Some("pandas"), + argument_type: "NUMBER(10, 2)", + expected_error: None, + }, + FrameArgumentTestCase { + description: "a scalar UDF has no frame", + dataframe: None, + argument_type: "NUMBER(10, 2)", + expected_error: None, + }, + ]; + + for test_case in test_cases { + let arguments: Vec<(String, String)> = vec![ + ("ts".to_owned(), "FLOAT".to_owned()), + ("amount".to_owned(), test_case.argument_type.to_owned()), + ]; + assert_eq!( + frame_argument_error( + "Python function functions/python/smooth.py", + test_case.dataframe, + &arguments + ) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs b/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs index 48e2d9152d..defb691353 100644 --- a/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs +++ b/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs @@ -20,6 +20,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("description", text(" Order rows. ")), ]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { arguments: vec![named(" raw_status ", "raw_status", "STRING")], returns: Some(FunctionReturns::Table(vec![named("id", "id", "INTEGER")])), @@ -37,6 +38,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("packages", HeaderValue::Sequence(vec![text("numpy")])), ]), language: FunctionLanguage::Python, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Type("STRING".to_owned())), runtime_version: Some("3.12".to_owned()), @@ -49,6 +51,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re description: "a missing return type is raised before the arguments", header: header(&[("arguments", HeaderValue::Other)]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { failure: failure( HeaderStage::Start, @@ -67,6 +70,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ), ]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { arguments: vec![named("id", "id", "INTEGER")], failure: failure( @@ -80,6 +84,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re description: "a blank SQL return type is a shape error after the arguments", header: header(&[("returns", text(" ")), ("tags", HeaderValue::Other)]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Table(Vec::new())), failure: failure( @@ -103,6 +108,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re )]), )]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Table(vec![named("id", "id", "INTEGER")])), failure: failure( @@ -116,6 +122,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re description: "an empty table is Python's column error", header: header(&[("returns", map(&[("table", map(&[]))]))]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Table(Vec::new())), failure: failure( @@ -137,6 +144,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("entry_point", text("label")), ]), language: FunctionLanguage::Python, + table_function: false, expected_header: FunctionHeader { failure: failure( HeaderStage::Start, @@ -153,6 +161,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("packages", HeaderValue::Other), ]), language: FunctionLanguage::Python, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Type("STRING".to_owned())), runtime_version: Some("3.12".to_owned()), @@ -172,6 +181,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("packages", HeaderValue::Sequence(vec![text("")])), ]), language: FunctionLanguage::Python, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Type("STRING".to_owned())), runtime_version: Some("3.12".to_owned()), @@ -192,6 +202,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("description", HeaderValue::Other), ]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Type("STRING".to_owned())), failure: failure( @@ -208,6 +219,7 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ("description", HeaderValue::Other), ]), language: FunctionLanguage::Sql, + table_function: false, expected_header: FunctionHeader { returns: Some(FunctionReturns::Type("STRING".to_owned())), failure: failure( @@ -217,11 +229,79 @@ fn given_function_headers_when_parsing_then_python_values_and_first_error_are_re ..empty_header() }, }, + FunctionHeaderTestCase { + description: "a Python table function maps each return column to its type", + header: header(&[ + ( + "returns", + map(&[(" ts ", text(" FLOAT ")), ("value", text("FLOAT"))]), + ), + ("runtime_version", text("3.12")), + ("entry_point", text("main")), + ]), + language: FunctionLanguage::Python, + table_function: true, + expected_header: FunctionHeader { + returns: Some(FunctionReturns::Table(vec![ + named(" ts ", "ts", "FLOAT"), + named("value", "value", "FLOAT"), + ])), + runtime_version: Some("3.12".to_owned()), + entry_point: Some("main".to_owned()), + ..empty_header() + }, + }, + FunctionHeaderTestCase { + description: "a Python table function with a scalar return type", + header: header(&[("returns", text("FLOAT"))]), + language: FunctionLanguage::Python, + table_function: true, + expected_header: FunctionHeader { + returns: Some(FunctionReturns::Table(Vec::new())), + failure: failure( + HeaderStage::Returns, + "Python function file functions/f.sql returns must map each column name to its type", + ), + ..empty_header() + }, + }, + FunctionHeaderTestCase { + description: "a Python table function with no return columns", + header: header(&[("returns", map(&[]))]), + language: FunctionLanguage::Python, + table_function: true, + expected_header: FunctionHeader { + returns: Some(FunctionReturns::Table(Vec::new())), + failure: failure( + HeaderStage::Returns, + "Python function file functions/f.sql returns must map each column name to its type", + ), + ..empty_header() + }, + }, + FunctionHeaderTestCase { + description: "a Python scalar UDF with column returns still needs a type string", + header: header(&[("returns", map(&[("ts", text("FLOAT"))]))]), + language: FunctionLanguage::Python, + table_function: false, + expected_header: FunctionHeader { + failure: failure( + HeaderStage::Start, + "Python function file functions/f.sql must declare returns", + ), + ..empty_header() + }, + }, ]; for test_case in test_cases { assert_eq!( - parse_function_header(&test_case.header, test_case.language, "functions/f.sql"), + parse_function_header( + &test_case.header, + test_case.language, + test_case.table_function, + "functions/f.sql" + ), test_case.expected_header, "{}", test_case.description diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_table_function_partition_error.rs b/crates/sqlbuild-attachments/src/functions/tests/test_table_function_partition_error.rs new file mode 100644 index 0000000000..86effcc911 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_table_function_partition_error.rs @@ -0,0 +1,51 @@ +use std::collections::HashSet; + +use crate::functions::main::table_function_partition_error::table_function_partition_error; +use crate::functions::tests::test_types::TableFunctionPartitionTestCase; + +#[test] +fn given_table_function_calls_when_checking_partitions_then_first_unpartitioned_call_is_reported() { + let whole_partition: HashSet = HashSet::from(["smooth_readings".to_owned()]); + let test_cases = [ + TableFunctionPartitionTestCase { + description: "a partitioned call and other functions' calls pass", + calls: vec![ + ("table_fn", "smooth_readings", true), + ("table_fn", "order_window", false), + ("ref", "readings", false), + ], + expected_error: None, + }, + TableFunctionPartitionTestCase { + description: "an unpartitioned call names the clause to add", + calls: vec![ + ("table_fn", "order_window", false), + ("table_fn", "smooth_readings", false), + ], + expected_error: Some( + "Model file models/report.sql calls Python table function 'smooth_readings' \ + without OVER (PARTITION BY ...); it receives one whole partition per call, so \ + add OVER (PARTITION BY ) after the call", + ), + }, + TableFunctionPartitionTestCase { + description: "a udf call of the same name is not a table function call", + calls: vec![("udf", "smooth_readings", false)], + expected_error: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + table_function_partition_error( + "Model file models/report.sql", + &test_case.calls, + &whole_partition + ) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_types.rs b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs index 4d84c817a7..e093668401 100644 --- a/crates/sqlbuild-attachments/src/functions/tests/test_types.rs +++ b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs @@ -6,6 +6,7 @@ pub(super) struct FunctionHeaderTestCase { pub(super) description: &'static str, pub(super) header: Vec<(String, HeaderValue)>, pub(super) language: FunctionLanguage, + pub(super) table_function: bool, pub(super) expected_header: FunctionHeader, } @@ -21,6 +22,19 @@ pub(super) struct FunctionReferenceTestCase { pub(super) expected_error: Option<&'static str>, } +pub(super) struct FrameArgumentTestCase { + pub(super) description: &'static str, + pub(super) dataframe: Option<&'static str>, + pub(super) argument_type: &'static str, + pub(super) expected_error: Option<&'static str>, +} + +pub(super) struct TableFunctionPartitionTestCase { + pub(super) description: &'static str, + pub(super) calls: Vec<(&'static str, &'static str, bool)>, + pub(super) expected_error: Option<&'static str>, +} + pub(super) struct TableFunctionArityTestCase { pub(super) description: &'static str, pub(super) calls: Vec<(&'static str, &'static str, Option)>, diff --git a/crates/sqlbuild-cache/src/stage_capture/constants.rs b/crates/sqlbuild-cache/src/stage_capture/constants.rs index 8b773cc563..1785c3a9a9 100644 --- a/crates/sqlbuild-cache/src/stage_capture/constants.rs +++ b/crates/sqlbuild-cache/src/stage_capture/constants.rs @@ -135,6 +135,7 @@ pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 15] = [ "runtime_version", "entry_point", "packages", + "dataframe", "declaration_usages", "tags", "description", @@ -159,6 +160,7 @@ pub const PROJECTED_ATTRIBUTES: [(&str, &[&str]); 15] = [ "runtime_version", "entry_point", "packages", + "dataframe", "tags", "description", ], diff --git a/crates/sqlbuild-config/src/project/_helpers/defaults.rs b/crates/sqlbuild-config/src/project/_helpers/defaults.rs index 07474bafe8..86e1bb42e9 100644 --- a/crates/sqlbuild-config/src/project/_helpers/defaults.rs +++ b/crates/sqlbuild-config/src/project/_helpers/defaults.rs @@ -4,8 +4,8 @@ use crate::models::ConfigValue; use crate::project::_helpers::durations::parse_duration; use crate::project::_helpers::fields::{ bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_batch_size, - optional_contract_policy, optional_cursor_start, optional_int, optional_mapping, - optional_non_negative_int, optional_nullable_bool, optional_strict_int, + optional_choice, optional_contract_policy, optional_cursor_start, optional_int, + optional_mapping, optional_non_negative_int, optional_nullable_bool, optional_strict_int, optional_templated_bool, optional_text_field, string_sequence, value_or_none, }; use crate::project::_helpers::help::{change_policy_problem, change_policy_toml_help}; @@ -13,9 +13,9 @@ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validat use crate::project::_helpers::sections::item_timeout; use crate::project::constants::{ BATCH_CONCURRENCY_KEY, CHANGE_POLICY_KEYS, CONTRACT_POLICIES, DEFAULTS_KEYS, DISABLED_VALUE, - GLOB_SEGMENTS, INHERIT_VALUE, MATERIALIZATION_DEFAULT_KEYS, MAX_MICROBATCHES_KEY, - MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, RETENTION_MATERIALIZATIONS, TABLE_TYPES, - TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, + FUNCTION_DATAFRAMES, GLOB_SEGMENTS, INHERIT_VALUE, MATERIALIZATION_DEFAULT_KEYS, + MAX_MICROBATCHES_KEY, MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, RETENTION_MATERIALIZATIONS, + TABLE_TYPES, TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -149,6 +149,15 @@ pub(crate) fn load_defaults(payload: Option<&ConfigValue>, file: ConfigFile) -> "contract", optional_contract_policy(mapping, "contract", &CONTRACT_POLICIES)?, )); + fields.push(( + "function_dataframe", + Built::Value(ConfigValue::String(optional_choice( + mapping, + "function_dataframe", + &FUNCTION_DATAFRAMES, + FUNCTION_DATAFRAMES[0], + )?)), + )); fields.extend(optional_text_fields( mapping, &[ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 466c57c3fc..d444929a7c 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -81,7 +81,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 19] = [ IGNORE_AUDIT_FAILURES_KEY, RECORD_RUN_HOST_KEY, ]; -pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ +pub(crate) const DEFAULTS_KEYS: [&str; 34] = [ "append_cursor_inclusive", "batch_concurrency", "batch_size", @@ -96,6 +96,7 @@ pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "database", "full_refresh", "function_database", + "function_dataframe", "function_schema", "incremental_mode", "incremental_strategy", @@ -310,6 +311,8 @@ pub(crate) const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; pub(crate) const FUTURE_CURSOR_ACTIONS: [&str; 2] = ["cap", "error"]; pub(crate) const MICROBATCH_LIMIT_ACTIONS: [&str; 2] = ["error", "warn"]; pub(crate) const CONTRACT_POLICIES: [&str; 2] = ["none", "enforced"]; +/// The frame libraries a Python table function's partition arrives in, the default first. +pub(crate) const FUNCTION_DATAFRAMES: [&str; 2] = ["polars", "pandas"]; pub(crate) const TABLE_TYPES: [&str; 2] = ["permanent", "transient"]; pub(crate) const CONFIRMATION_POLICIES: [&str; 3] = ["deny", "require_confirmation", "allow"]; pub(crate) const INHERIT_VALUE: &str = "inherit"; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 84dbfb4f22..23319a9e06 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -78,6 +78,20 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "Expected 'pause_dynamic' to be a boolean when provided", )), }, + ProjectFailureTestCase { + description: "a project may switch Python table functions to pandas frames", + project: "name = 'orders'\nadapter = 'snowflake'\n[defaults]\nfunction_dataframe = 'pandas'\n", + local: "", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "a function frame library is polars or pandas", + project: "name = 'orders'\nadapter = 'snowflake'\n[defaults]\nfunction_dataframe = 'arrow'\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'function_dataframe' to be one of: polars, pandas", + )), + }, ProjectFailureTestCase { description: "an invalid dynamic section fails the project load", project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\nheartbeat_grace = 'soon'\n", diff --git a/crates/sqlbuild-project/src/models.rs b/crates/sqlbuild-project/src/models.rs index 3d6e8c124f..da0e41dd44 100644 --- a/crates/sqlbuild-project/src/models.rs +++ b/crates/sqlbuild-project/src/models.rs @@ -13,13 +13,15 @@ use sqlbuild_core::json::models::JsonValue; use crate::types::{InferredRow, NamedTypeRow, ObjectKeyRow, ProjectHost, SourceNamespace}; -/// One reference a resource's SQL makes: kind, name, package and call argument count. +/// One reference a resource's SQL makes: kind, name, package and its call's arguments and partition. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ReferenceRow { pub kind: String, pub name: String, pub package: Option, pub call_argument_count: Option, + /// Whether a table function call is followed by `OVER (PARTITION BY ...)`. + pub call_partitioned: bool, } /// A compiled relation location's parts. @@ -134,12 +136,14 @@ pub struct SourceRecord { pub host: H::Source, } -/// A Python function's runtime, entry point and packages. +/// A Python function's runtime, entry point, packages and, for a table function, its frame. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PythonRuntime { pub runtime_version: Option, pub entry_point: Option, pub packages: Vec, + /// A table function's partition frame library, `"polars"` or `"pandas"`. + pub dataframe: Option, } /// What the function loop attached for one SQL or Python function. diff --git a/crates/sqlbuild-pysource/src/udf_functions/_helpers/dataframes.rs b/crates/sqlbuild-pysource/src/udf_functions/_helpers/dataframes.rs new file mode 100644 index 0000000000..9f58f8951a --- /dev/null +++ b/crates/sqlbuild-pysource/src/udf_functions/_helpers/dataframes.rs @@ -0,0 +1,93 @@ +//! The dataframe library a function's first parameter annotation names, through module imports. + +use ruff_python_ast::{Alias, Expr, Stmt, StmtFunctionDef}; + +use crate::udf_functions::constants::{DATAFRAME_CLASS, PANDAS_MODULE, POLARS_MODULE}; +use crate::udf_functions::models::DataFrameLibrary; + +/// `pd.DataFrame` after `import pandas as pd`, or `Frame` after `from polars import DataFrame as Frame`. +pub(crate) fn parameter_dataframe( + body: &[Stmt], + function: &StmtFunctionDef, +) -> Option { + let parameters = &function.parameters; + let first = parameters + .posonlyargs + .iter() + .chain(¶meters.args) + .next()?; + match first.parameter.annotation.as_deref()? { + Expr::Attribute(attribute) if attribute.attr.as_str() == DATAFRAME_CLASS => { + let Expr::Name(module) = attribute.value.as_ref() else { + return None; + }; + imported_module(body, module.id.as_str()) + } + Expr::Name(name) => imported_class(body, name.id.as_str()), + _ => None, + } +} + +/// The library `import pandas` or `import pandas as ` binds to `alias`. +fn imported_module(body: &[Stmt], alias: &str) -> Option { + body.iter() + .find_map(|statement| import_library(statement, alias)) +} + +/// The library one `import` statement binds to `alias`. +fn import_library(statement: &Stmt, alias: &str) -> Option { + let Stmt::Import(import) = statement else { + return None; + }; + import + .names + .iter() + .find_map(|name| module_library(name, alias)) +} + +/// The library one `import` name binds to `alias`. +fn module_library(name: &Alias, alias: &str) -> Option { + if bound_name(name) != alias { + return None; + } + library(name.name.as_str()) +} + +/// The library `from pandas import DataFrame` or `... import DataFrame as ` binds to `alias`. +fn imported_class(body: &[Stmt], alias: &str) -> Option { + body.iter() + .find_map(|statement| import_from_library(statement, alias)) +} + +/// The library one `from ... import` statement binds to `alias`. +fn import_from_library(statement: &Stmt, alias: &str) -> Option { + let Stmt::ImportFrom(import) = statement else { + return None; + }; + let module: &str = import.module.as_ref()?.as_str(); + import + .names + .iter() + .find_map(|name| class_library(name, module, alias)) +} + +/// The library one `from import` name binds to `alias` when it is the frame class. +fn class_library(name: &Alias, module: &str, alias: &str) -> Option { + if bound_name(name) != alias || name.name.as_str() != DATAFRAME_CLASS { + return None; + } + library(module) +} + +/// The name an imported alias binds in the module. +fn bound_name(name: &Alias) -> &str { + name.asname.as_ref().unwrap_or(&name.name).as_str() +} + +fn library(module: &str) -> Option { + match module { + PANDAS_MODULE => Some(DataFrameLibrary::Pandas), + POLARS_MODULE => Some(DataFrameLibrary::Polars), + _ => None, + } +} diff --git a/crates/sqlbuild-pysource/src/udf_functions/_helpers/decorators.rs b/crates/sqlbuild-pysource/src/udf_functions/_helpers/decorators.rs index d1a5eb3df8..36b215bc8d 100644 --- a/crates/sqlbuild-pysource/src/udf_functions/_helpers/decorators.rs +++ b/crates/sqlbuild-pysource/src/udf_functions/_helpers/decorators.rs @@ -1,4 +1,4 @@ -//! The `@udf(...)` decorator, its imports and the function docstring, read from the ruff tree. +//! The `@udf(...)` and `@table_fn(...)` decorators, their imports and the function docstring. use ruff_python_ast::{Decorator, Expr, ExprCall, Identifier, Keyword, Stmt, StmtFunctionDef}; use ruff_text_size::Ranged; @@ -6,11 +6,13 @@ use ruff_text_size::Ranged; use crate::cpython_tree::main::source_position::source_position; use crate::cpython_tree::models::SourceLines; use crate::udf_functions::_helpers::docstrings::clean_docstring; -use crate::udf_functions::constants::{UDF_DECORATOR_NAME, UDF_IMPORT_MODULES}; -use crate::udf_functions::models::UdfKeyword; +use crate::udf_functions::constants::{ + TABLE_FN_DECORATOR_NAME, UDF_DECORATOR_NAME, UDF_IMPORT_MODULES, +}; +use crate::udf_functions::models::{FunctionDecorator, UdfKeyword}; -/// The decorator's call when it is `udf(...)` or `.udf(...)`. -pub(crate) fn udf_call(decorator: &Decorator) -> Option<&ExprCall> { +/// The decorator's call and kind when it is `udf(...)`, `table_fn(...)` or `.` either. +pub(crate) fn udf_call(decorator: &Decorator) -> Option<(&ExprCall, FunctionDecorator)> { let Expr::Call(call) = &decorator.expression else { return None; }; @@ -19,10 +21,14 @@ pub(crate) fn udf_call(decorator: &Decorator) -> Option<&ExprCall> { Expr::Attribute(attribute) => attribute.attr.as_str(), _ => return None, }; - (name == UDF_DECORATOR_NAME).then_some(call) + match name { + UDF_DECORATOR_NAME => Some((call, FunctionDecorator::Udf)), + TABLE_FN_DECORATOR_NAME => Some((call, FunctionDecorator::TableFn)), + _ => None, + } } -/// `from sqlbuild import udf` or `from sqlbuild.functions import udf`, at any relative level. +/// `from sqlbuild import udf` or `from sqlbuild.functions import table_fn`, at any relative level. pub(crate) fn is_udf_import(statement: &Stmt) -> bool { let Stmt::ImportFrom(import) = statement else { return false; @@ -31,10 +37,9 @@ pub(crate) fn is_udf_import(statement: &Stmt) -> bool { .module .as_ref() .is_some_and(|module| UDF_IMPORT_MODULES.contains(&module.as_str())) - && import - .names - .iter() - .any(|alias| alias.name.as_str() == UDF_DECORATOR_NAME) + && import.names.iter().any(|alias| { + [UDF_DECORATOR_NAME, TABLE_FN_DECORATOR_NAME].contains(&alias.name.as_str()) + }) } /// `ast.get_docstring(function)`: a leading string expression statement, cleaned. diff --git a/crates/sqlbuild-pysource/src/udf_functions/_helpers/mod.rs b/crates/sqlbuild-pysource/src/udf_functions/_helpers/mod.rs index fe671dd97b..7350b898d5 100644 --- a/crates/sqlbuild-pysource/src/udf_functions/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/udf_functions/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod dataframes; pub(crate) mod decorators; pub(crate) mod docstrings; diff --git a/crates/sqlbuild-pysource/src/udf_functions/constants.rs b/crates/sqlbuild-pysource/src/udf_functions/constants.rs index b3fe20adda..ba02d5778e 100644 --- a/crates/sqlbuild-pysource/src/udf_functions/constants.rs +++ b/crates/sqlbuild-pysource/src/udf_functions/constants.rs @@ -2,7 +2,14 @@ /// The decorator name a UDF function is marked with, bare or as an attribute. pub const UDF_DECORATOR_NAME: &str = "udf"; -/// The modules whose `udf` import the warehouse body drops. +/// The decorator name a Python table function is marked with, bare or as an attribute. +pub const TABLE_FN_DECORATOR_NAME: &str = "table_fn"; +/// The class name a dataframe annotation names, as `.DataFrame` or an imported `DataFrame`. +pub(crate) const DATAFRAME_CLASS: &str = "DataFrame"; +/// The modules whose `DataFrame` selects pandas or polars. +pub(crate) const PANDAS_MODULE: &str = "pandas"; +pub(crate) const POLARS_MODULE: &str = "polars"; +/// The modules whose `udf` and `table_fn` imports the warehouse body drops. pub const UDF_IMPORT_MODULES: [&str; 2] = ["sqlbuild", "sqlbuild.functions"]; /// The keywords `@udf(...)` accepts. pub const UDF_KEYS: [&str; 9] = [ diff --git a/crates/sqlbuild-pysource/src/udf_functions/main/udf_function.rs b/crates/sqlbuild-pysource/src/udf_functions/main/udf_function.rs index 946a6a4698..d7c14ab7fb 100644 --- a/crates/sqlbuild-pysource/src/udf_functions/main/udf_function.rs +++ b/crates/sqlbuild-pysource/src/udf_functions/main/udf_function.rs @@ -1,17 +1,18 @@ -//! Find the `@udf(...)` function of a Python function file without importing it. +//! Find the `@udf(...)` or `@table_fn(...)` function of a Python function file without importing it. use ruff_python_ast::{ExprCall, Stmt, StmtFunctionDef}; use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; use crate::cpython_tree::models::ParsedSource; +use crate::udf_functions::_helpers::dataframes::parameter_dataframe; use crate::udf_functions::_helpers::decorators::{docstring, is_udf_import, udf_call, udf_keyword}; -use crate::udf_functions::models::{UdfFunction, UdfKeyword, UdfShape}; +use crate::udf_functions::models::{FunctionDecorator, UdfFunction, UdfKeyword, UdfShape}; -/// The `@udf(...)` functions of `contents`, or `None` when the source does not parse. +/// The `@udf(...)` or `@table_fn(...)` function of `contents`, or `None` when it does not parse. #[must_use] pub fn udf_function(contents: &str, python_version: (u8, u8)) -> Option { let parsed: ParsedSource = parse_cpython_module(contents, python_version)?; - let mut matches: Vec<(usize, &StmtFunctionDef, &ExprCall)> = Vec::new(); + let mut matches: Vec<(usize, &StmtFunctionDef, (&ExprCall, FunctionDecorator))> = Vec::new(); for (index, statement) in parsed.module.body.iter().enumerate() { let Stmt::FunctionDef(function) = statement else { continue; @@ -24,7 +25,7 @@ pub fn udf_function(contents: &str, python_version: (u8, u8)) -> Option return Some(UdfShape::Missing), [single] => *single, _ => return Some(UdfShape::Multiple), @@ -37,6 +38,8 @@ pub fn udf_function(contents: &str, python_version: (u8, u8)) -> Option, pub keywords: Vec, /// Whether the decorator call passes positional arguments. pub has_positional_arguments: bool, @@ -26,7 +45,7 @@ pub struct UdfFunction { pub removed_statements: Vec, } -/// How many top-level functions a module decorates with `@udf(...)`. +/// How many top-level functions a module decorates with `@udf(...)` or `@table_fn(...)`. #[derive(Clone, Debug, PartialEq, Eq)] pub enum UdfShape { Missing, diff --git a/crates/sqlbuild-pysource/src/udf_functions/tests/test_udf_function.rs b/crates/sqlbuild-pysource/src/udf_functions/tests/test_udf_function.rs index d369166ca1..f0a7b6d126 100644 --- a/crates/sqlbuild-pysource/src/udf_functions/tests/test_udf_function.rs +++ b/crates/sqlbuild-pysource/src/udf_functions/tests/test_udf_function.rs @@ -1,5 +1,5 @@ use crate::udf_functions::main::udf_function::udf_function; -use crate::udf_functions::models::{UdfFunction, UdfShape}; +use crate::udf_functions::models::{DataFrameLibrary, FunctionDecorator, UdfFunction, UdfShape}; use crate::udf_functions::tests::helpers::keyword; use crate::udf_functions::tests::test_types::UdfFunctionTestCase; @@ -12,6 +12,8 @@ fn given_function_file_when_reading_udf_then_matches_cpython_facts() { contents: "from sqlbuild import udf\nimport os\n\n@other\n@udf(returns=\"INTEGER\",\n arguments=[\n \"x INTEGER\",\n ])\ndef add_one(x):\n \"\"\"Add one.\n\n Indented detail.\n Trailing.\n \"\"\"\n return x + 1\n", expected_shape: Some(UdfShape::Single(UdfFunction { name: "add_one".to_owned(), + decorator: FunctionDecorator::Udf, + dataframe: None, keywords: vec![keyword(Some("returns"), 5), keyword(Some("arguments"), 6)], has_positional_arguments: false, docstring: Some("Add one.\n\n Indented detail.\nTrailing.".to_owned()), @@ -25,6 +27,8 @@ fn given_function_file_when_reading_udf_then_matches_cpython_facts() { contents: "import sqlbuild\n@sqlbuild.udf(**extra)\nasync def f():\n\t\"\"\"\n\tTabbed.\n\t More.\n\t\"\"\"\n", expected_shape: Some(UdfShape::Single(UdfFunction { name: "f".to_owned(), + decorator: FunctionDecorator::Udf, + dataframe: None, keywords: vec![keyword(None, 2)], has_positional_arguments: false, docstring: Some("Tabbed.\n More.".to_owned()), @@ -43,6 +47,56 @@ fn given_function_file_when_reading_udf_then_matches_cpython_facts() { contents: "@udf()\ndef f():\n pass\n\n@udf()\ndef g():\n pass\n", expected_shape: Some(UdfShape::Multiple), }, + UdfFunctionTestCase { + description: "a table function with a polars alias annotation drops both imports", + contents: "import polars as pl\nfrom sqlbuild.functions import table_fn\n\n@table_fn(returns={\"a\": \"FLOAT\"})\ndef main(rows: pl.DataFrame) -> pl.DataFrame:\n return rows\n", + expected_shape: Some(UdfShape::Single(UdfFunction { + name: "main".to_owned(), + decorator: FunctionDecorator::TableFn, + dataframe: Some(DataFrameLibrary::Polars), + keywords: vec![keyword(Some("returns"), 4)], + has_positional_arguments: false, + docstring: None, + statement_index: 2, + removed_decorators: vec![0], + removed_statements: vec![1], + })), + }, + UdfFunctionTestCase { + description: "a pandas DataFrame imported by name, with polars also imported", + contents: "import polars\nfrom pandas import DataFrame as Frame\nfrom sqlbuild import table_fn\n\n@table_fn()\ndef main(rows: Frame) -> polars.DataFrame:\n return rows\n", + expected_shape: Some(UdfShape::Single(UdfFunction { + name: "main".to_owned(), + decorator: FunctionDecorator::TableFn, + dataframe: Some(DataFrameLibrary::Pandas), + keywords: Vec::new(), + has_positional_arguments: false, + docstring: None, + statement_index: 3, + removed_decorators: vec![0], + removed_statements: vec![2], + })), + }, + UdfFunctionTestCase { + description: "an unannotated or quoted parameter names no library", + contents: "import pandas as pd\nfrom sqlbuild.functions import table_fn\n\n@table_fn()\ndef main(rows: \"pd.DataFrame\"):\n return rows\n", + expected_shape: Some(UdfShape::Single(UdfFunction { + name: "main".to_owned(), + decorator: FunctionDecorator::TableFn, + dataframe: None, + keywords: Vec::new(), + has_positional_arguments: false, + docstring: None, + statement_index: 2, + removed_decorators: vec![0], + removed_statements: vec![1], + })), + }, + UdfFunctionTestCase { + description: "a udf and a table function in one module", + contents: "@udf()\ndef f():\n pass\n\n@table_fn()\ndef g(rows):\n pass\n", + expected_shape: Some(UdfShape::Multiple), + }, UdfFunctionTestCase { description: "invalid syntax", contents: "def f(:\n", diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs index 1eda7ce0f7..78a4ed72bb 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compile_records/constants.rs @@ -391,6 +391,7 @@ const R_COMPILED_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("runtime_version", FieldDefault::None), FieldSpec::new("entry_point", FieldDefault::None), FieldSpec::new("packages", FieldDefault::EmptyTuple), + FieldSpec::new("dataframe", FieldDefault::None), FieldSpec::new("tags", FieldDefault::EmptyTuple), FieldSpec::new("description", FieldDefault::None), ], @@ -679,7 +680,7 @@ const R_DISCOVERED_MODEL_SCHEMA_FILE: RecordSpec = RecordSpec { }; const R_DISCOVERED_PYTHON_FUNCTION_FILE: RecordSpec = RecordSpec { name: "DiscoveredPythonFunctionFile", - doc: "A discovered Python function file and its parsed UDF metadata.", + doc: "A discovered Python function file and its parsed UDF or table function metadata.", fields: &[ FieldSpec::required("file_path"), FieldSpec::required("relative_path"), @@ -687,6 +688,8 @@ const R_DISCOVERED_PYTHON_FUNCTION_FILE: RecordSpec = RecordSpec { FieldSpec::required("header_values"), FieldSpec::required("entry_point"), FieldSpec::required("body_python"), + FieldSpec::new("table_function", FieldDefault::False), + FieldSpec::new("dataframe", FieldDefault::None), ], }; const R_DISCOVERED_SCHEMA_FILE: RecordSpec = RecordSpec { @@ -1354,6 +1357,7 @@ const R_COMPILE_SQL_REFERENCE: RecordSpec = RecordSpec { FieldSpec::required("ref_name"), FieldSpec::new("ref_package", FieldDefault::None), FieldSpec::new("call_argument_count", FieldDefault::None), + FieldSpec::new("call_partitioned", FieldDefault::False), ], }; const R_COMPILED_LINEAGE_SOURCE_FACT: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 58bc8be8fa..33a1cd0d9a 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -339,6 +339,7 @@ const R_DEFAULTS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("seed_schema", FieldDefault::None), FieldSpec::new("function_database", FieldDefault::None), FieldSpec::new("function_schema", FieldDefault::None), + FieldSpec::new("function_dataframe", FieldDefault::Text("polars")), FieldSpec::new("incremental_strategy", FieldDefault::None), FieldSpec::new("incremental_mode", FieldDefault::None), FieldSpec::new("microbatch_strategy", FieldDefault::None), diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 6c629f4d2b..1d3f2765ea 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -749,6 +749,7 @@ const R_FUNCTION_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("runtime_version", FieldDefault::None), FieldSpec::new("entry_point", FieldDefault::None), FieldSpec::new("packages", FieldDefault::EmptyTuple), + FieldSpec::new("dataframe", FieldDefault::None), FieldSpec::new("previous_query_sql", FieldDefault::None), FieldSpec::new( "reason", diff --git a/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs index 00daa4b9bf..c9908f9f54 100644 --- a/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs +++ b/crates/sqlbuild-python-compile/src/compile_command/_helpers/artifacts/target_writer.rs @@ -189,6 +189,7 @@ fn function_sql(adapter: &Bound<'_, PyAny>, function: &Bound<'_, PyAny>) -> PyRe "runtime_version", "entry_point", "packages", + "dataframe", ] { arguments.push((field, native_field(function, field)?)); } diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs index aaa7af2b28..0d740fa2ad 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/model/model_loop_inputs.rs @@ -73,6 +73,7 @@ pub(crate) fn reference_row(reference: &SqlReference) -> ReferenceRow { name: reference.name.clone(), package: reference.package.clone(), call_argument_count: reference.call_argument_count, + call_partitioned: reference.call_partitioned, } } diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs index b5e80f1bc3..f87709ab9d 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/attachment.rs @@ -208,6 +208,7 @@ pub(crate) fn attach_resources<'py>( target: target_namespace(&target_config)?, inherit: python_functions_inherit_default_namespace, kind: function_kind.unbind(), + dataframe: native_field(&defaults, "function_dataframe")?.extract()?, }, )?; let mut source_rows: Vec = Vec::new(); diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs index 71c32a6ba7..32029bd826 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/function_loop.rs @@ -18,9 +18,11 @@ use sqlbuild_analysis::compiled_project::models::ScanFaultOrigin; use sqlbuild_analysis::compiler::main::function_type_validation::function_type_error; use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_attachments::functions::main::frame_argument_error::frame_argument_error; use sqlbuild_attachments::functions::main::function_reference_error::function_reference_error; use sqlbuild_attachments::functions::main::resolve_function_namespace::resolve_function_namespace; use sqlbuild_attachments::functions::main::table_function_arity_error::table_function_arity_error; +use sqlbuild_attachments::functions::main::table_function_partition_error::table_function_partition_error; use sqlbuild_attachments::functions::models::{ FunctionHeader, FunctionLanguage, FunctionNamespace, FunctionReturns, HeaderStage, KnownResources, NamedType, NamespaceInputs, @@ -44,6 +46,7 @@ use sqlbuild_project::models::PythonRuntime; use sqlbuild_project::models::ReferenceRow; use sqlbuild_project::types::NamedTypeRow; use sqlbuild_python_base::compile_context::main::delta::record_source_reads::record_source_reads; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::attachments::main::parsed_header::parsed_header; use sqlbuild_python_front::model_config::models::NativeModelValidator; use sqlbuild_python_project::analysis_session::main::query_columns_error::query_columns_error; @@ -72,6 +75,7 @@ struct FunctionScope<'a, 'py> { target: Option, inherit: bool, function_kind: Bound<'py, PyAny>, + dataframe: String, } /// Attach every SQL function, then every Python function, in discovery order. @@ -89,6 +93,7 @@ pub(crate) fn attached_functions<'py>( target, inherit, kind, + dataframe, } = settings; let defaults = ( optional_template(context, default_database, "default function database")?, @@ -115,6 +120,7 @@ pub(crate) fn attached_functions<'py>( }), inherit, function_kind: kind.into_bound(py), + dataframe, }; let (sql_rows, python_rows) = rows; let mut names: HashSet = HashSet::new(); @@ -153,25 +159,14 @@ fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult = header_values.into_bound(py); let label: String = format!("SQL function {relative_path}"); let header: FunctionHeader = - parsed_header(&header_values, FunctionLanguage::Sql, &relative_path); + parsed_header(&header_values, FunctionLanguage::Sql, false, &relative_path); raise_stage(context, &header, HeaderStage::Start)?; let arguments: Vec = expanded_types(context, &header.arguments, |raw| { format!("{label} argument '{raw}' type") })?; raise_stage(context, &header, HeaderStage::Arguments)?; - let (returns, return_columns): (String, Vec) = match &header.returns { - Some(FunctionReturns::Table(columns)) => ( - TABLE_RETURNS.to_owned(), - expanded_types(context, columns, |raw| { - format!("{label} return column '{raw}' type") - })?, - ), - Some(FunctionReturns::Type(text)) => ( - header_template(context, text, &format!("{label} returns"))?, - Vec::new(), - ), - None => (String::from("None"), Vec::new()), - }; + let (returns, return_columns): (String, Vec) = + expanded_returns(context, &header, &label)?; raise_stage(context, &header, HeaderStage::Returns)?; let namespace: FunctionNamespace = function_namespace(scope, &header_values, &label, FunctionLanguage::Sql)?; @@ -249,19 +244,26 @@ fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult< } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("Python function {relative_path}"); - let header: FunctionHeader = - parsed_header(&header_values, FunctionLanguage::Python, &relative_path); + let file: &Bound<'_, PyAny> = function_file.bind(py); + let table_function: bool = native_field(file, "table_function")?.extract()?; + let dataframe: Option = table_function + .then(|| native_field(file, "dataframe")?.extract::>()) + .transpose()? + .map(|annotated| annotated.unwrap_or_else(|| scope.dataframe.clone())); + let header: FunctionHeader = parsed_header( + &header_values, + FunctionLanguage::Python, + table_function, + &relative_path, + ); raise_stage(context, &header, HeaderStage::Start)?; let arguments: Vec = expanded_types(context, &header.arguments, |raw| { format!("{label} argument '{raw}' type") })?; raise_stage(context, &header, HeaderStage::Arguments)?; - let returns: String = match &header.returns { - Some(FunctionReturns::Type(text)) => { - header_template(context, text, &format!("{label} returns"))? - } - _ => String::from("None"), - }; + let (returns, return_columns): (String, Vec) = + expanded_returns(context, &header, &label)?; + raise_stage(context, &header, HeaderStage::Returns)?; raise_stage(context, &header, HeaderStage::PythonValues)?; let namespace: FunctionNamespace = function_namespace(scope, &header_values, &label, FunctionLanguage::Python)?; @@ -274,7 +276,10 @@ fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult< ), )?; if scope.validate { - validated_types(scope, &label, &arguments, (&returns, &[]))?; + validated_types(scope, &label, &arguments, (&returns, &return_columns))?; + } + if let Some(message) = frame_argument_error(&label, dataframe.as_deref(), &arguments) { + return Err(compile_error(context.classes, &message, None)); } raise_stage(context, &header, HeaderStage::Metadata)?; Ok(AttachedFunction { @@ -284,13 +289,14 @@ fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult< name, arguments, returns, - return_columns: Vec::new(), + return_columns, body, references: Vec::new(), python: Some(PythonRuntime { runtime_version: header.runtime_version, entry_point: header.entry_point, packages: header.packages, + dataframe, }), tags: header.tags, description: header.description, @@ -301,6 +307,27 @@ fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult< }) } +/// The scalar return type, or `TABLE` and its columns, with their templates expanded. +fn expanded_returns( + context: &LoopContext<'_, '_>, + header: &FunctionHeader, + label: &str, +) -> PyResult<(String, Vec)> { + Ok(match &header.returns { + Some(FunctionReturns::Table(columns)) => ( + TABLE_RETURNS.to_owned(), + expanded_types(context, columns, |raw| { + format!("{label} return column '{raw}' type") + })?, + ), + Some(FunctionReturns::Type(text)) => ( + header_template(context, text, &format!("{label} returns"))?, + Vec::new(), + ), + None => (String::from("None"), Vec::new()), + }) +} + /// `_raise_header_failure`: the header's error when it is raised at `stage`. fn raise_stage( context: &LoopContext<'_, '_>, @@ -576,7 +603,17 @@ fn destination(name: &str, namespace: &FunctionNamespace, fingerprint: bool) -> } } -/// `validate_table_function_call_arities` over each model's, then each function's, calls. +/// The Python table functions, which receive one whole partition per call. +pub(crate) fn whole_partition_functions<'a>( + functions: impl Iterator, +) -> HashSet { + functions + .filter(|function| function.python.is_some() && !function.return_columns.is_empty()) + .map(|function| function.name.clone()) + .collect() +} + +/// Call arities and whole-partition clauses over each model's, then each function's, calls. pub(crate) fn checked_table_function_arities( context: &LoopContext<'_, '_>, models: &[(String, Vec)], @@ -587,6 +624,7 @@ pub(crate) fn checked_table_function_arities( .filter(|function| !function.return_columns.is_empty()) .map(|function| (function.name.clone(), function.arguments.len())) .collect(); + let whole_partition: HashSet = whole_partition_functions(functions.iter()); let mut owners: Vec<(String, &[ReferenceRow])> = Vec::new(); for (path, references) in models { owners.push((format!("Model file {path}"), references)); @@ -612,6 +650,20 @@ pub(crate) fn checked_table_function_arities( if let Some(message) = table_function_arity_error(&owner, &calls, &counts) { return Err(compile_error(context.classes, &message, None)); } + let partitions: Vec<(&str, &str, bool)> = references + .iter() + .map(|reference| { + ( + reference.kind.as_str(), + reference.name.as_str(), + reference.call_partitioned, + ) + }) + .collect(); + if let Some(message) = table_function_partition_error(&owner, &partitions, &whole_partition) + { + return Err(compile_error(context.classes, &message, None)); + } } Ok(()) } diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs index bfb9223a6d..e0375c34cf 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/resource_attachment.rs @@ -1,11 +1,12 @@ //! The model loop's attachment of the project's sources, functions, SQL tests and audits. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::input_loops::_helpers::resources::audit_loop::model_owner; use crate::input_loops::_helpers::resources::audit_loop::resource_owner; +use crate::input_loops::_helpers::resources::function_loop::whole_partition_functions; use crate::input_loops::models::AttachmentOwner; use crate::input_loops::models::NativeModelLoop; use sqlbuild_project::models::ReferenceRow; @@ -103,9 +104,19 @@ pub(crate) fn attach_sql_tests( ) }) .collect(); - model_loop - .get() - .run_sql_tests(host.bind(py), (&files, &scenarios), &settings, arities) + let whole_partition: HashSet = whole_partition_functions( + project + .borrow() + .functions() + .iter() + .map(|record| &record.attached), + ); + model_loop.get().run_sql_tests( + host.bind(py), + (&files, &scenarios), + &settings, + (arities, whole_partition), + ) } /// Attach singular, then resource-attached, audits; return views and override warnings. diff --git a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs index 486708799d..37a66c1c5b 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/_helpers/resources/sql_test_loop.rs @@ -26,6 +26,7 @@ use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; use sqlbuild_attachments::functions::main::table_function_arity_error::table_function_arity_error; +use sqlbuild_attachments::functions::main::table_function_partition_error::table_function_partition_error; use sqlbuild_attachments::test_parameters::main::parameter_references::parameter_references; use sqlbuild_attachments::test_targets::main::unknown_test_target::unknown_test_target; use sqlbuild_attachments::test_targets::models::{TargetCatalog, TestTargets}; @@ -80,6 +81,8 @@ pub(crate) struct TestScope<'a, 'py> { pub(crate) catalog: TargetCatalog, pub(crate) functions: HashSet, pub(crate) table_function_arities: HashMap, + /// The Python table functions, whose calls need `OVER (PARTITION BY ...)`. + pub(crate) whole_partition: HashSet, } /// One test case expanded before extraction. @@ -464,6 +467,23 @@ impl<'py> TestScope<'_, 'py> { ) { return Err(self.error(&message)); } + let partitions: Vec<(&str, &str, bool)> = references + .iter() + .map(|reference| { + ( + reference.kind, + reference.name.as_str(), + reference.call_partitioned, + ) + }) + .collect(); + if let Some(message) = table_function_partition_error( + &format!("SQL test file {relative_path}"), + &partitions, + &self.whole_partition, + ) { + return Err(self.error(&message)); + } } let kind: &str = if table_function { TABLE_FUNCTION_KIND diff --git a/crates/sqlbuild-python-compile/src/input_loops/models.rs b/crates/sqlbuild-python-compile/src/input_loops/models.rs index 2eb18b2ce3..e30e1f2fe6 100644 --- a/crates/sqlbuild-python-compile/src/input_loops/models.rs +++ b/crates/sqlbuild-python-compile/src/input_loops/models.rs @@ -401,7 +401,7 @@ impl NativeModelLoop { host: &Bound<'py, ConstantRendering>, rows: (&[TestFileRow], &[ScenarioRow]), settings: &SqlTestSettings, - table_function_arities: HashMap, + (table_function_arities, whole_partition): (HashMap, HashSet), ) -> PyResult { let py: Python<'py> = host.py(); let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; @@ -457,6 +457,7 @@ impl NativeModelLoop { }, functions: known.functions.clone(), table_function_arities, + whole_partition, }; let tests: Vec> = scope.attached(rows.0)?; let scenarios: Vec> = scope.attached_scenarios(rows.1)?; diff --git a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs index d7c6d9fda0..9ebd58c72a 100644 --- a/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs +++ b/crates/sqlbuild-python-front/src/attachments/main/parsed_header.rs @@ -11,6 +11,7 @@ use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, pub fn parsed_header( header_values: &Bound<'_, PyDict>, language: FunctionLanguage, + table_function: bool, relative_path: &str, ) -> FunctionHeader { let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); @@ -23,5 +24,5 @@ pub fn parsed_header( } header.push((name, header_value(&value))); } - parse_function_header(&header, language, relative_path) + parse_function_header(&header, language, table_function, relative_path) } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs index 42d3474bf8..ec57878ee2 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/python_files/python_function_files.rs @@ -1,5 +1,6 @@ //! Python function files discovered natively, with CPython as the authority on syntax and values. +use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PySyntaxError, PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; @@ -8,7 +9,9 @@ use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_pysource::udf_functions::constants::UDF_KEYS; use sqlbuild_pysource::udf_functions::main::udf_function::udf_function; -use sqlbuild_pysource::udf_functions::models::{UdfFunction, UdfShape}; +use sqlbuild_pysource::udf_functions::models::{ + DataFrameLibrary, FunctionDecorator, UdfFunction, UdfShape, +}; use crate::discovery::_helpers::failures::raised::{failure_error, read_error}; use crate::discovery::_helpers::project::project_files::path_object; @@ -24,6 +27,17 @@ const AST_MODULE: &str = "ast"; const DESCRIPTION_KEY: &str = "description"; const ENTRY_POINT_KEY: &str = "entry_point"; const UDF_STATEMENT: &str = "@udf(...)"; +const TABLE_FN_STATEMENT: &str = "@table_fn(...)"; +const DATAFRAME_PANDAS: &str = "pandas"; +const DATAFRAME_POLARS: &str = "polars"; + +/// The decorator call as messages name it. +fn decorator_call(function: &UdfFunction) -> &'static str { + match function.decorator { + FunctionDecorator::Udf => UDF_STATEMENT, + FunctionDecorator::TableFn => TABLE_FN_STATEMENT, + } +} /// One file's paths and text, as its messages and record name them. struct FunctionSource<'py> { @@ -68,7 +82,7 @@ fn cpython_module<'py>(source: &FunctionSource<'py>) -> PyResult( source: &FunctionSource<'py>, function: &UdfFunction, @@ -80,7 +94,10 @@ fn keyword_values<'py>( let nodes: Bound<'py, PyAny> = native_field(call, "keywords")?; for (keyword, node) in function.keywords.iter().zip(nodes.try_iter()?) { let Some(name) = &keyword.name else { - let detail: String = format!("{UDF_STATEMENT} must not use **kwargs expansion"); + let detail: String = format!( + "{} must not use **kwargs expansion", + decorator_call(function) + ); return Err(parse_error(source, &detail, None)); }; match literal_eval.call1((native_field(&node?, "value")?,)) { @@ -89,7 +106,10 @@ fn keyword_values<'py>( if error.is_instance_of::(py) || error.is_instance_of::(py) => { - let detail: String = format!("{UDF_STATEMENT} value for '{name}' must be literal"); + let detail: String = format!( + "{} value for '{name}' must be literal", + decorator_call(function) + ); return Err(parse_error(source, &detail, Some(error))); } Err(error) => return Err(error), @@ -98,7 +118,7 @@ fn keyword_values<'py>( Ok(values) } -/// The unsupported-keys error of the first keywords `@udf(...)` does not accept, if any. +/// The unsupported-keys error of the first keywords the decorator does not accept, if any. fn unsupported_keys(source: &FunctionSource<'_>, function: &UdfFunction) -> PyResult<()> { let unsupported: Vec<(&str, usize)> = function .keywords @@ -113,7 +133,8 @@ fn unsupported_keys(source: &FunctionSource<'_>, function: &UdfFunction) -> PyRe let keys: Vec<&str> = unsupported.iter().map(|(name, _)| *name).collect(); let supported: Vec = UDF_KEYS.iter().map(|key| (*key).to_owned()).collect(); let message: String = format!( - "{UDF_STATEMENT} in '{}:{line}' has unsupported keys: {}", + "{} in '{}:{line}' has unsupported keys: {}", + decorator_call(function), source.file_path.str()?, keys.join(", ") ); @@ -125,7 +146,7 @@ fn unsupported_keys(source: &FunctionSource<'_>, function: &UdfFunction) -> PyRe Err(PyErr::from_value(error)) } -/// `ast.unparse` of the module without its `udf` imports and the function's `@udf` decorator. +/// `ast.unparse` of the module without its `udf` and `table_fn` imports and the function's decorator. fn warehouse_body<'py>( module: &Bound<'py, PyAny>, function: &UdfFunction, @@ -161,10 +182,8 @@ fn warehouse_body<'py>( .call_method0("strip") } -/// `parse_python_function`: `(header_values, entry_point, body_python)` of one function file. -fn parsed_function<'py>( - source: &FunctionSource<'py>, -) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyAny>, Bound<'py, PyAny>)> { +/// `parse_python_function`: one function file's record, with its header, entry point, body and kind. +fn parsed_record<'py>(source: &FunctionSource<'py>) -> PyResult> { let py: Python<'py> = source.file_path.py(); let module: Bound<'py, PyAny> = cpython_module(source)?; let version = py.version_info(); @@ -174,11 +193,12 @@ fn parsed_function<'py>( })?; let function: UdfFunction = match shape { UdfShape::Missing => { - let detail = "must define exactly one function decorated with @udf(...)"; + let detail = + "must define exactly one function decorated with @udf(...) or @table_fn(...)"; return Err(parse_error(source, detail, None)); } UdfShape::Multiple => { - let detail = "must define only one function decorated with @udf(...)"; + let detail = "must define only one function decorated with @udf(...) or @table_fn(...)"; return Err(parse_error(source, detail, None)); } UdfShape::Single(function) => function, @@ -189,12 +209,15 @@ fn parsed_function<'py>( .removed_decorators .first() .copied() - .ok_or_else(|| compiler_error("a @udf(...) function without its decorator"))?; + .ok_or_else(|| compiler_error("a Python function file function without its decorator"))?; let call: Bound<'py, PyAny> = native_field(&statement, "decorator_list")?.get_item(decorator)?; let values: Bound<'py, PyDict> = keyword_values(source, &function, &call)?; if function.has_positional_arguments { - let detail: String = format!("{UDF_STATEMENT} must use keyword arguments only"); + let detail: String = format!( + "{} must use keyword arguments only", + decorator_call(&function) + ); return Err(parse_error(source, &detail, None)); } unsupported_keys(source, &function)?; @@ -217,10 +240,31 @@ fn parsed_function<'py>( Err(_) => None, }; let Some(entry_point) = stripped else { - let detail: String = format!("{UDF_STATEMENT} entry_point must be a non-empty string"); + let detail: String = format!( + "{} entry_point must be a non-empty string", + decorator_call(&function) + ); return Err(parse_error(source, &detail, None)); }; - Ok((values, entry_point, warehouse_body(&module, &function)?)) + let table_function: bool = function.decorator == FunctionDecorator::TableFn; + let dataframe: Option<&str> = function.dataframe.map(|library| match library { + DataFrameLibrary::Pandas => DATAFRAME_PANDAS, + DataFrameLibrary::Polars => DATAFRAME_POLARS, + }); + new_record_with( + py, + RecordId::DiscoveredPythonFunctionFile, + &[ + ("file_path", source.file_path.clone()), + ("relative_path", source.relative_path.clone()), + ("contents", source.contents.clone().into_any()), + ("header_values", values.into_any()), + ("entry_point", entry_point), + ("body_python", warehouse_body(&module, &function)?), + ("table_function", table_function.into_bound_py_any(py)?), + ("dataframe", dataframe.into_bound_py_any(py)?), + ], + ) } pub(crate) fn function_record<'py>( @@ -242,17 +286,5 @@ pub(crate) fn function_record<'py>( relative_path, contents: PyString::new(py, &contents), }; - let (header_values, entry_point, body_python) = parsed_function(&source)?; - new_record_with( - py, - RecordId::DiscoveredPythonFunctionFile, - &[ - ("file_path", source.file_path.clone()), - ("relative_path", source.relative_path.clone()), - ("contents", source.contents.clone().into_any()), - ("header_values", header_values.into_any()), - ("entry_point", entry_point), - ("body_python", body_python), - ], - ) + parsed_record(&source) } diff --git a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs index f82f3a1eff..71a77638ac 100644 --- a/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs +++ b/crates/sqlbuild-python-front/src/model_config/_helpers/model_nodes/model_validator.rs @@ -72,7 +72,14 @@ pub(crate) fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult = file?; + let name: String = model_stem(&file)?; + if native_field(&file, "table_function")?.extract::()? { + table_functions.insert(name.clone()); + } + functions.insert(name); + } Ok(ResourceNames { models, seeds, diff --git a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs index 1caa7ef112..186403ba1c 100644 --- a/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs +++ b/crates/sqlbuild-python-front/src/sqltext/_helpers/sql_references.rs @@ -150,6 +150,7 @@ pub(crate) fn scanned<'py>( "call_argument_count", text(py, reference.call_argument_count)?, ), + ("call_partitioned", text(py, reference.call_partitioned)?), ], ) }) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs index f47081b779..ba41a54bb6 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/clone_plans.rs @@ -473,6 +473,7 @@ fn clone_function_entries<'py>( ("runtime_version", field("runtime_version")?), ("entry_point", field("entry_point")?), ("packages", field("packages")?), + ("dataframe", field("dataframe")?), ], )?; entries.set_item( diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index 452b3f06f4..253d95cab2 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -31,6 +31,7 @@ use sqlbuild_python_project::planner_scope::main::build_source_load_entries::bui use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; use sqlbuild_python_project::planner_strategy::main::planning_context::planning_context; use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; +use sqlbuild_python_project::resource_views::main::function_reads::dataframe::dataframe; use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; use sqlbuild_python_project::resource_views::main::function_reads::fingerprint_destination::fingerprint_destination; use sqlbuild_python_project::resource_views::main::function_reads::language::language; @@ -424,6 +425,7 @@ pub(crate) fn function_entry<'py>( ), ("entry_point", entry_point(function)?), ("packages", packages(function)?), + ("dataframe", dataframe(function)?), ("previous_query_sql", previous_query_sql), ("reason", reason), ], diff --git a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs index 21bbcc1793..8d527153eb 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/_helpers/metadata.rs @@ -31,6 +31,7 @@ use sqlbuild_python_project::planner_identity::main::baselined_function_definiti use sqlbuild_python_project::planner_identity::main::function_definition_hash::function_definition_hash; use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::dataframe::dataframe; use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; @@ -319,6 +320,10 @@ pub(crate) fn function_local_hashes<'py>( payload.set_item("runtime_version", runtime_version(&function)?)?; payload.set_item("entry_point", entry_point(&function)?)?; payload.set_item("packages", packages(&function)?)?; + let frame: Bound<'py, PyAny> = dataframe(&function)?; + if !frame.is_none() { + payload.set_item("dataframe", frame)?; + } let mut local_hash: String = stable_identity_hash(&sorted_dumps(payload.as_any(), SPACED, true)?); if recorded_function_unchanged(&function, (&name, dialect), recorded)? { diff --git a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs index 14ddb5ab0f..725c6af40e 100644 --- a/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs +++ b/crates/sqlbuild-python-planner/src/planner_identity/main/compiled_function_fingerprint_sql.rs @@ -7,6 +7,7 @@ use sqlbuild_analysis::planner_identity::main::function_fingerprint_sql::functio use sqlbuild_analysis::planner_identity::models::FunctionFingerprintFields; use sqlbuild_python_project::resource_views::main::function_reads::arguments::arguments; use sqlbuild_python_project::resource_views::main::function_reads::body_sql::body_sql; +use sqlbuild_python_project::resource_views::main::function_reads::dataframe::dataframe; use sqlbuild_python_project::resource_views::main::function_reads::entry_point::entry_point; use sqlbuild_python_project::resource_views::main::function_reads::packages::packages; use sqlbuild_python_project::resource_views::main::function_reads::return_columns::return_columns; @@ -31,6 +32,7 @@ pub fn compiled_function_fingerprint_sql(function: &Bound<'_, PyAny>) -> PyResul .try_iter()? .map(|package| package?.extract::()) .collect::>()?, + dataframe: dataframe(function)?.extract()?, body_sql: body_sql(function)?.extract()?, })) } diff --git a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs index 8db885ebe1..8afacd93d7 100644 --- a/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs +++ b/crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/refs.rs @@ -15,10 +15,13 @@ use pyo3::pyfunction; use pyo3::types::PyDict; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::sqltext::models::SqlCodeScanner; +use sqlbuild_sqltext::resolution::main::over_clause_open::over_clause_open; use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans::reference_spans}; +use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax; #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -149,6 +152,9 @@ fn resolve_function_calls( return Ok(query_sql.to_owned()); } let scan = SqlCodeScanner::new(query_sql, None, "SQL reference resolution".to_owned()); + let dialect_name: Option = dialect.syntax("sql_analysis_dialect_name")?.extract()?; + let line_comments: Vec = + dialect_lexical_syntax(dialect_name.as_deref().unwrap_or_default()).line_comment_prefixes; let mut output = String::new(); let mut last = 0; let mut code_index = 0; @@ -189,6 +195,13 @@ fn resolve_function_calls( .char_indices() .nth(end + 1) .map_or(query_sql.len(), |(index, _)| index); + let end = match table + .then(|| over_clause_open(query_sql, end, &line_comments)) + .flatten() + { + Some(open) => over_clause_close(query_sql, open, &scanner, py)?, + None => end, + }; let hook = if table { "render_table_function_call" } else { @@ -217,6 +230,20 @@ fn resolve_function_calls( Ok(output) } +/// The end of the `OVER (...)` clause whose `(` is at byte `open`. +fn over_clause_close( + query_sql: &str, + open: usize, + scanner: &SqlCodeScanner, + py: Python<'_>, +) -> PyResult { + let close: usize = scanner.matching_paren(py, query_sql[..open].chars().count())?; + Ok(query_sql + .char_indices() + .nth(close + 1) + .map_or(query_sql.len(), |(index, _)| index)) +} + #[pyfunction] #[pyo3(signature = (*, query_sql, function_locations, dialect))] pub(crate) fn resolve_udf_references( diff --git a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs index 60a765269f..c234e4799f 100644 --- a/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_scenario/_helpers/entries.rs @@ -134,6 +134,7 @@ pub(crate) fn function_entry<'py>( "runtime_version", "entry_point", "packages", + "dataframe", ] { fields.push((name, native_field(function, name)?)); } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs index 252124f7ec..2c1ea85c08 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/reference_row_tuple.rs @@ -10,6 +10,7 @@ pub fn reference_row_tuple(row: ReferenceRow) -> ReferenceTuple { name, package, call_argument_count, + .. } = row; ReferenceTuple { kind, diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs index 4d18108958..654a5e9a9a 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -113,7 +113,7 @@ impl<'py> pyo3::FromPyObject<'py> for FunctionRow { } } -/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. +/// Adapter, validation, raw default and target namespaces, inheritance, resource kind and frame. #[derive(Debug)] pub struct FunctionSettings { pub adapter_name: String, @@ -122,11 +122,13 @@ pub struct FunctionSettings { pub target: Option<(Option, Option)>, pub inherit: bool, pub kind: Py, + /// The frame library of a Python table function whose parameter names none. + pub dataframe: String, } impl<'py> pyo3::FromPyObject<'py> for FunctionSettings { fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { - let (adapter_name, validate, defaults, target, inherit, kind): FunctionSettingsTuple = + let (adapter_name, validate, defaults, target, inherit, kind, dataframe): FunctionSettingsTuple = pyo3::types::PyAnyMethods::extract(value)?; Ok(Self { adapter_name, @@ -135,6 +137,7 @@ impl<'py> pyo3::FromPyObject<'py> for FunctionSettings { target, inherit, kind, + dataframe, }) } } diff --git a/crates/sqlbuild-python-project/src/compiled_project/types.rs b/crates/sqlbuild-python-project/src/compiled_project/types.rs index 9254d9b6ea..320377048c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/types.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/types.rs @@ -34,6 +34,7 @@ pub type FunctionSettingsTuple = ( Option<(Option, Option)>, bool, Py, + String, ); /// The tuple Python sends for a `TestBlockRow`, in field order. diff --git a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs index 0046c3ab7b..f70306ae22 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/model_input_view.rs @@ -187,5 +187,6 @@ pub(crate) fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult( "packages" => this .packages(py) .and_then(|value| value.into_bound_py_any(py)), + "dataframe" => this + .dataframe(py) + .and_then(|value| value.into_bound_py_any(py)), "tags" => this.tags(py).and_then(|value| value.into_bound_py_any(py)), "description" => this .description(py) @@ -390,6 +393,9 @@ pub(crate) fn read_native_function_input<'py>( "packages" => this .packages(py) .and_then(|value| value.into_bound_py_any(py)), + "dataframe" => this + .dataframe(py) + .and_then(|value| value.into_bound_py_any(py)), "tags" => this.tags(py).and_then(|value| value.into_bound_py_any(py)), "description" => this .description(py) diff --git a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs index 6c1f3cd3ef..4f2c377d84 100644 --- a/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs +++ b/crates/sqlbuild-python-project/src/compiled_views/main/model_inputs/reference_tuple.rs @@ -21,6 +21,7 @@ pub fn reference_tuple( name, package, call_argument_count: count, + call_partitioned: partitioned, } in rows { let fields: Bound<'_, PyDict> = PyDict::new(py); @@ -28,6 +29,7 @@ pub fn reference_tuple( fields.set_item("ref_name", name)?; fields.set_item("ref_package", package)?; fields.set_item("call_argument_count", count)?; + fields.set_item("call_partitioned", partitioned)?; references.push(classes.sql_reference.call((), Some(&fields))?); } Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) diff --git a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs index f6e5c2a22c..738e5f3bbf 100644 --- a/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs +++ b/crates/sqlbuild-python-project/src/resource_views/_helpers/function_views.rs @@ -217,6 +217,17 @@ macro_rules! function_getters { }) } + #[getter] + pub(crate) fn dataframe(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function + .attached + .python + .as_ref() + .and_then(|python| python.dataframe.clone())) + }) + } + #[getter] pub(crate) fn tags<'py>(&self, py: Python<'py>) -> PyResult> { read_function(&self.owner, self.index, py, |function, py| { diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/dataframe.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/dataframe.rs new file mode 100644 index 0000000000..cafc80f1a2 --- /dev/null +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/dataframe.rs @@ -0,0 +1,17 @@ +//! The compiled function's `dataframe`, as its Python attribute reads it. + +use crate::resource_views::_helpers::function_reads::native_function; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The compiled function's `dataframe`, as its Python attribute reads it. +pub fn dataframe<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + if let Some(view) = native_function(function)? { + return view + .dataframe(py) + .and_then(|value| value.into_bound_py_any(py)); + } + native_field(function, "dataframe") +} diff --git a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs index 04cf303460..f9f5c4d5fb 100644 --- a/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs +++ b/crates/sqlbuild-python-project/src/resource_views/main/function_reads/mod.rs @@ -1,5 +1,6 @@ pub mod arguments; pub mod body_sql; +pub mod dataframe; pub mod deps; pub mod destination; pub mod entry_point; diff --git a/crates/sqlbuild-python-project/src/resource_views/models.rs b/crates/sqlbuild-python-project/src/resource_views/models.rs index a0afc09f48..fc984e3934 100644 --- a/crates/sqlbuild-python-project/src/resource_views/models.rs +++ b/crates/sqlbuild-python-project/src/resource_views/models.rs @@ -240,6 +240,7 @@ impl NativeCompiledFunction { self.runtime_version(py)?.into_bound_py_any(py)?, self.entry_point(py)?.into_bound_py_any(py)?, self.packages(py)?.into_any(), + self.dataframe(py)?.into_bound_py_any(py)?, self.tags(py)?.into_any(), self.description(py)?.into_bound_py_any(py)?, ]) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs index a0ab6f6cc7..c55efaaf38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -367,6 +367,7 @@ fn function_statements( ("runtime_version", native_field(entry, "runtime_version")?), ("entry_point", native_field(entry, "entry_point")?), ("packages", native_field(entry, "packages")?), + ("dataframe", native_field(entry, "dataframe")?), ], )?; Ok(FunctionStatements { diff --git a/crates/sqlbuild-sqltext/src/resolution/constants.rs b/crates/sqlbuild-sqltext/src/resolution/constants.rs index 7f086a7682..899dd8e608 100644 --- a/crates/sqlbuild-sqltext/src/resolution/constants.rs +++ b/crates/sqlbuild-sqltext/src/resolution/constants.rs @@ -1,6 +1,8 @@ //! Alias boundaries retained from the planner resolution contract. pub(super) const EXTERNAL_REFERENCE: &str = "dbt_ref"; +/// The keyword of a partition clause that follows a table function call. +pub(super) const OVER_KEYWORD: &str = "OVER"; pub(super) const REF_CLAUSES: &[&str] = &[ "WHERE", "JOIN", "INNER", "LEFT", "RIGHT", "FULL", "CROSS", "ON", "GROUP", "ORDER", "HAVING", diff --git a/crates/sqlbuild-sqltext/src/resolution/main/code_start.rs b/crates/sqlbuild-sqltext/src/resolution/main/code_start.rs new file mode 100644 index 0000000000..ea25f54368 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/code_start.rs @@ -0,0 +1,24 @@ +//! The next code after whitespace and SQL comments. + +/// The first code byte after `start` past whitespace, `/* */` and `line_comments` comments. +#[must_use] +pub fn code_start(sql: &str, start: usize, line_comments: &[String]) -> Option { + let mut position: usize = start; + loop { + let tail: &str = sql.get(position..)?; + position += tail.len() - tail.trim_start().len(); + let rest: &str = &sql[position..]; + if line_comments + .iter() + .any(|prefix| rest.starts_with(prefix.as_str())) + { + position = rest + .find('\n') + .map_or(sql.len(), |newline| position + newline + 1); + } else if let Some(body) = rest.strip_prefix("/*") { + position += 2 + body.find("*/")? + 2; + } else { + return Some(position); + } + } +} diff --git a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs index 037d057ac1..8b4ac9056e 100644 --- a/crates/sqlbuild-sqltext/src/resolution/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/resolution/main/mod.rs @@ -1,7 +1,9 @@ //! Public SQL resolution decisions. +pub mod code_start; pub mod column_name_key; pub mod cursor_subquery; +pub mod over_clause_open; pub mod quoted_name; pub mod ref_alias; pub mod reference_spans; diff --git a/crates/sqlbuild-sqltext/src/resolution/main/over_clause_open.rs b/crates/sqlbuild-sqltext/src/resolution/main/over_clause_open.rs new file mode 100644 index 0000000000..e1bea22cb8 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/resolution/main/over_clause_open.rs @@ -0,0 +1,17 @@ +//! The `OVER (...)` clause that follows a table function call. + +use crate::resolution::constants::OVER_KEYWORD; +use crate::resolution::main::code_start::code_start; + +/// The `(` of an `OVER (` clause after `start`, with whitespace or comments before each word. +#[must_use] +pub fn over_clause_open(sql: &str, start: usize, line_comments: &[String]) -> Option { + let keyword: usize = code_start(sql, start, line_comments)?; + let after: usize = keyword + OVER_KEYWORD.len(); + let word: &str = sql.get(keyword..after)?; + if !word.eq_ignore_ascii_case(OVER_KEYWORD) { + return None; + } + let open: usize = code_start(sql, after, line_comments)?; + sql[open..].starts_with('(').then_some(open) +} diff --git a/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs b/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs index 00204fa275..8fdc39c747 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/_helpers/scan.rs @@ -1,9 +1,11 @@ //! The general reference scan of Python `extract_sql_references`, reproduced exactly. +use crate::resolution::main::code_start::code_start; +use crate::resolution::main::over_clause_open::over_clause_open; use crate::sql_references::constants::{ - DBT_REFERENCE_KIND, NAME_QUOTE_BYTES, NON_CODE_START_BYTES, PLACEHOLDER_NAMES, QUOTE_BYTES, - QUOTED_NAME_MINIMUM_BYTES, REFERENCE_CONTEXT, REFERENCE_PREFIXES, TABLE_FUNCTION_CALL_CONTEXT, - TABLE_FUNCTION_REFERENCE_KIND, + DBT_REFERENCE_KIND, NAME_QUOTE_BYTES, NON_CODE_START_BYTES, PARTITION_CLAUSE_WORDS, + PLACEHOLDER_NAMES, QUOTE_BYTES, QUOTED_NAME_MINIMUM_BYTES, REFERENCE_CONTEXT, + REFERENCE_PREFIXES, TABLE_FUNCTION_CALL_CONTEXT, TABLE_FUNCTION_REFERENCE_KIND, }; use crate::sql_references::models::{InvalidReferenceCall, ReferenceScan, SqlReference}; use crate::sql_references::types::ReferencePrefix; @@ -104,6 +106,7 @@ fn parse_reference( return Ok((Parsed::Invalid(invalid), close + 1)); }; let mut call_argument_count = None; + let mut call_partitioned = false; if kind == TABLE_FUNCTION_REFERENCE_KIND { let suffix = python_whitespace_end(text, close + 1); if sql.get(suffix) != Some(&b'(') { @@ -126,6 +129,7 @@ fn parse_reference( } let suffix_close = matching_paren(sql, suffix, syntax, TABLE_FUNCTION_CALL_CONTEXT)?; call_argument_count = Some(split_arguments(text, suffix + 1, suffix_close, syntax)?.len()); + call_partitioned = partition_clause_follows(text, suffix_close + 1, syntax); } Ok(( Parsed::Reference(SqlReference { @@ -133,6 +137,7 @@ fn parse_reference( name, package, call_argument_count, + call_partitioned, }), close + 1, )) @@ -345,6 +350,36 @@ fn empty_argument() -> String { format!("{REFERENCE_CONTEXT} contains an empty argument") } +/// Whether `OVER (PARTITION BY` follows `start`, in any case, with whitespace or comments between. +fn partition_clause_follows(text: &str, start: usize, syntax: &LexicalSyntax) -> bool { + let comments: &[String] = &syntax.line_comment_prefixes; + let Some(open) = over_clause_open(text, start, comments) else { + return false; + }; + let mut position: usize = open + 1; + for word in PARTITION_CLAUSE_WORDS { + let Some(word_start) = code_start(text, position, comments) else { + return false; + }; + position = word_start; + let Some(candidate) = text.get(position..position + word.len()) else { + return false; + }; + if !candidate.eq_ignore_ascii_case(word) { + return false; + } + position += word.len(); + if text[position..] + .chars() + .next() + .is_some_and(|character| character.is_alphanumeric() || character == '_') + { + return false; + } + } + true +} + /// Python `_skip_whitespace`, which skips `str.isspace()` characters. fn python_whitespace_end(text: &str, start: usize) -> usize { text[start..] diff --git a/crates/sqlbuild-sqltext/src/sql_references/constants.rs b/crates/sqlbuild-sqltext/src/sql_references/constants.rs index c0812999ab..e4ebddb3be 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/constants.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/constants.rs @@ -32,3 +32,6 @@ pub(crate) const PLACEHOLDER_NAMES: [(&str, &[&str]); 6] = [ ("table_fn", &["function_name"]), ("dbt_ref", &["package_name", "model_name"]), ]; + +/// The words that open a partition clause inside `OVER (`, in order. +pub(crate) const PARTITION_CLAUSE_WORDS: [&str; 2] = ["PARTITION", "BY"]; diff --git a/crates/sqlbuild-sqltext/src/sql_references/models.rs b/crates/sqlbuild-sqltext/src/sql_references/models.rs index c6029db8d5..d66556ec63 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/models.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/models.rs @@ -9,6 +9,8 @@ pub struct SqlReference { pub package: Option, /// Top-level arguments of a table function's call suffix; `None` for other kinds. pub call_argument_count: Option, + /// Whether a table function's call is followed by `OVER (PARTITION BY ...)`. + pub call_partitioned: bool, } /// One reference call compile cannot replace, with Python's P012 message and help. diff --git a/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs index 75f366d800..92b076f20d 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/tests/helpers.rs @@ -27,6 +27,7 @@ pub(crate) fn reference(kind: &'static str, name: &str) -> SqlReference { name: name.to_string(), package: None, call_argument_count: None, + call_partitioned: false, } } @@ -37,6 +38,13 @@ pub(crate) fn table_function(name: &str, call_argument_count: usize) -> SqlRefer } } +pub(crate) fn partitioned_table_function(name: &str, call_argument_count: usize) -> SqlReference { + SqlReference { + call_partitioned: true, + ..table_function(name, call_argument_count) + } +} + pub(crate) fn dbt_reference(package: &str, name: &str) -> SqlReference { SqlReference { package: Some(package.to_string()), diff --git a/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs b/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs index 253bcc89d7..4078a738e0 100644 --- a/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs +++ b/crates/sqlbuild-sqltext/src/sql_references/tests/test_extract_sql_references.rs @@ -1,7 +1,7 @@ use crate::sql_references::main::extract_sql_references::extract_sql_references; use crate::sql_references::tests::helpers::{ - backslash_hash_comments, dbt_reference, extracted, failed, generic, invalid_call, reference, - rejected, table_function, + backslash_hash_comments, dbt_reference, extracted, failed, generic, invalid_call, + partitioned_table_function, reference, rejected, table_function, }; use crate::sql_references::tests::test_types::ExtractSqlReferencesTestCase; @@ -29,6 +29,33 @@ fn given_sql_when_extracting_references_then_matches_python_scanner() { syntax: generic(), expected_extraction: extracted(vec![table_function("expand_orders", 3)]), }, + ExtractSqlReferencesTestCase { + description: "a partition clause after the call marks it partitioned", + sql: "SELECT s.* FROM r, __table_fn(\"smooth\")(r.a, r.b)\n over ( partition\tby r.device_id ORDER BY r.ts) AS s", + syntax: generic(), + expected_extraction: extracted(vec![partitioned_table_function("smooth", 2)]), + }, + ExtractSqlReferencesTestCase { + description: "comments around the partition clause keep the call partitioned", + sql: "SELECT * FROM __table_fn(\"a\")(1) /* c */ OVER (-- key\n PARTITION /* x */ BY k), __table_fn(\"b\")(2) -- c\n OVER (PARTITION BY k)", + syntax: generic(), + expected_extraction: extracted(vec![ + partitioned_table_function("a", 1), + partitioned_table_function("b", 1), + ]), + }, + ExtractSqlReferencesTestCase { + description: "a dialect line comment before OVER keeps the call partitioned", + sql: "SELECT * FROM __table_fn(\"a\")(1) # c\n OVER (PARTITION BY k)", + syntax: backslash_hash_comments(), + expected_extraction: extracted(vec![partitioned_table_function("a", 1)]), + }, + ExtractSqlReferencesTestCase { + description: "an empty window or a following OVER word is not a partition clause", + sql: "SELECT * FROM __table_fn(\"a\")(1) OVER () , __table_fn(\"b\")(2) overflow (partition by x)", + syntax: generic(), + expected_extraction: extracted(vec![table_function("a", 1), table_function("b", 1)]), + }, ExtractSqlReferencesTestCase { description: "table function without call arguments", sql: "SELECT * FROM __table_fn(\"daily_orders\")()", diff --git a/fensu.toml b/fensu.toml index c282764ae4..a8cb2d83ed 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3473,7 +3473,7 @@ paths = ["crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_ reason = "Each builds its own index from its own node list; only the index-building loop is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs"] +paths = ["crates/sqlbuild-python-project/src/resource_views/main/audit_reads/always_run.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_kind.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/attached_target_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/definition_name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/description.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_limit.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/evidence_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/meta.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/references.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/scope_deps.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/severity.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/sql_body.rs", "crates/sqlbuild-python-project/src/resource_views/main/audit_reads/thresholds.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/arguments.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/body_sql.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/dataframe.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/fingerprint_destination.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/language.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/packages.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/relative_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/return_columns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/returns.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/runtime_version.rs", "crates/sqlbuild-python-project/src/resource_views/main/function_reads/source_file_path.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/external.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/seed_reads/schema_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/key.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/name.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_entry.rs", "crates/sqlbuild-python-project/src/resource_views/main/source_reads/source_file.rs"] reason = "Each getter reads its own attribute of a compiled resource, from the native view when the object is one and by attribute otherwise; only the view-or-attribute shape is alike." [[dupes.allowlist]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md index ef845f99e5..2829e1d9cf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/functions.md @@ -13,7 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/functions/ - References inside functions - Change propagation - Project layout -- Python UDF options +- Python function options - Adapter support Functions are SQL or Python definitions under `functions/` that SQLBuild compiles and deploys to the warehouse alongside your models. They participate in the DAG: if a function definition changes, SQLBuild redeploys it, and every model that calls it directly treats the change as a change to its own query. @@ -74,7 +74,11 @@ The function name is the file stem (filename without extension). `__udf()` resol ## Table functions -Table functions return multiple rows and columns. They are written in SQL under `functions/sql/` with a `returns table(...)` declaration: +Table functions return multiple rows and columns. Write them in SQL under `functions/sql/` with a `returns table(...)` declaration, or in Python under `functions/python/` with `@table_fn(...)`. + +### SQL table functions + +A SQL table function's body is a query: ```sql -- functions/sql/table_fn__customer_orders.sql @@ -102,6 +106,52 @@ FROM __ref("fact_orders") WHERE customer_id = p_customer_id ``` +### Python table functions + +A Python table function receives one whole partition of its input rows as a data frame and returns a data frame of rows. Use it for per-group numerics that SQL expresses badly, such as smoothing a series or fitting a curve per customer. Place it under `functions/python/`, with exactly one function decorated with `@table_fn(...)`: + +```python +# functions/python/smooth_payments.py +import polars as pl +from sqlbuild.functions import table_fn + +@table_fn( + arguments={"customer_id": "INTEGER", "paid_at": "FLOAT", "amount": "FLOAT"}, + returns={"customer_id": "INTEGER", "paid_at": "FLOAT", "amount_smoothed": "FLOAT"}, + runtime_version="3.12", +) +def main(payments: pl.DataFrame) -> pl.DataFrame: + """Payment amounts smoothed over each customer's last three payments.""" + return payments.sort("paid_at").select( + "customer_id", + "paid_at", + pl.col("amount").rolling_mean(3, min_samples=1).alias("amount_smoothed"), + ) +``` + +Call it with `OVER (PARTITION BY ...)` after the arguments. Each partition is one call of `main`: + +```sql +SELECT s.* +FROM __ref("payments") p, + __table_fn("smooth_payments")(p.customer_id, p.paid_at, p.amount) + OVER (PARTITION BY p.customer_id) s +``` + +- `main` is a plain function: its frame has one column per argument, named after the argument. Test it with pytest like any other Python code. +- The frame is a polars `DataFrame` by default. Annotate the parameter as a pandas `pd.DataFrame` to receive pandas, or set `function_dataframe = "pandas"` under [`[defaults]`](project-configuration.md#defaults) to make pandas the project default. An annotation wins over the default. SQLBuild reads it through the file's top-level `import pandas` or `from pandas import DataFrame`; an annotation in quotes or imported under `if TYPE_CHECKING:` is not read, so the function gets the default. +- SQLBuild generates the warehouse handler and adds the packages the frame needs: `pandas`, and `polars` and `pyarrow` for polars. +- The returned frame must have exactly the `returns` columns. Otherwise the call fails with an error naming the missing, extra and repeated columns. +- Rows arrive in no particular order. Sort inside `main` when order matters. + +| Problem | Fix | +|---------|-----| +| A call without `OVER (PARTITION BY ...)` fails compilation | Add `OVER (PARTITION BY )` after the call | +| A polars function's `NUMBER(p, s)` argument (or `DECIMAL`, `DEC`, `NUMERIC`) with a scale above zero fails compilation | Declare it as `FLOAT`, or as `INTEGER` when it has no fraction | +| `returns` is not a column-to-type mapping, so compilation fails | Map each output column to its type | + +Python table functions deploy on Snowflake only; other adapters reject them when the function is created (see [Adapter support](#adapter-support)). + ### Why table functions exist Table functions are designed as an alternative to final-layer views for cases where views don't push predicates efficiently. A view over `fact_orders` with a `WHERE customer_id = ?` filter may scan the entire table if the engine doesn't push the predicate down. A table function accepts the filter as an argument and guarantees the predicate is applied at execution time. @@ -115,7 +165,7 @@ SELECT customer_id, order_id FROM __table_fn("table_fn__customer_orders")(42) ``` -The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. +The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. A Python table function call also needs `OVER (PARTITION BY ...)`. The reference creates a typed dependency in the project DAG. Selecting a consuming model also selects the required function, function changes propagate to consumers, and cycles through models and functions are rejected. SQLBuild treats the call as an opaque relation for SQL analysis and uses the function's declared `returns table(...)` columns for star expansion and terminal lineage. @@ -161,19 +211,20 @@ functions/ udf__is_completed_order.sql # scalar SQL UDF table_fn__customer_orders.sql # table function (returns table) python/ - is_completed_order_py.py # scalar Python UDF + is_completed_order_py.py # scalar Python UDF (@udf) + smooth_payments.py # Python table function (@table_fn) ``` -## Python UDF options +## Python function options -The `@udf` decorator accepts these keyword arguments: +The `@udf` and `@table_fn` decorators accept these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| | `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | -| `returns` | Yes | SQL return type string | -| `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | +| `returns` | Yes | `@udf`: SQL return type string. `@table_fn`: dict mapping each output column to its SQL type | +| `runtime_version` | Yes | Python runtime version (e.g. `"3.11"`) | | `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | | `packages` | No | List of Python packages required at runtime | @@ -181,12 +232,11 @@ The `@udf` decorator accepts these keyword arguments: ## Adapter support -All four supported adapters implement SQL UDFs, Python UDFs, and table functions: - -| Feature | DuckDB | Snowflake | BigQuery | Databricks | -|---------|--------|-----------|----------|------------| -| SQL UDFs | Yes | Yes | Yes | Yes | -| Python UDFs | Yes | Yes | Yes | Yes | -| Table functions | Yes | Yes | Yes | Yes | +| Feature | DuckDB, MotherDuck | Snowflake | BigQuery | Databricks | PostgreSQL | SQL Server | ClickHouse | SQLite | +|---------|--------------------|-----------|----------|------------|------------|------------|------------|--------| +| SQL UDFs | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | +| Python UDFs | Yes | Yes | Yes | Yes | No | No | No | No | +| SQL table functions | Yes | Yes | Yes | Yes | No | No | No | No | +| Python table functions | No | Yes | No | No | No | No | No | No | -Future adapters may not support all function types. SQLBuild raises a clear error if a function type is unsupported by the configured adapter. +Creating a function the adapter doesn't support fails with an error naming the adapter and the function type. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index a15b81ec2b..2252f5bf37 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -419,6 +419,9 @@ instead. - Placement: `database`, `schema`, `seed_database`, `seed_schema`, `function_database`, `function_schema` +- Functions: `function_dataframe`, the frame library of Python table functions whose parameter + names none: `"polars"` (default) or `"pandas"` (see + [Python table functions](functions.md#python-table-functions)) - Materialization: `materialized`, `contract`, `full_refresh`, `replay_on_change`, `tags` - Incremental: `incremental_strategy`, `incremental_mode`, `merge_exclude_columns`, `append_cursor_inclusive`, `lookback`, `unaccounted_partition_policy` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing-reference.md index a5748e44ff..7599251f0c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing-reference.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing-reference.md @@ -192,8 +192,8 @@ __table_fn_expected__ AS ( ) ``` -Table function tests run after the function is created. Models can't depend on table functions, so -these tests check the function on its own. +Table function tests run after the function is created. Call a Python table function with +`OVER (PARTITION BY ...)` in the test, as in a model. ## Repeating test logic diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d3384c5a85..322b4b7f1a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -459,6 +459,8 @@ class FunctionInput(Protocol): @property def packages(self) -> tuple[str, ...]: ... @property + def dataframe(self) -> str | None: ... + @property def tags(self) -> tuple[str, ...]: ... @property def description(self) -> str | None: ... @@ -1616,6 +1618,8 @@ class NativeFunctionInput: @property def packages(self) -> tuple[str, ...]: ... @property + def dataframe(self) -> str | None: ... + @property def tags(self) -> tuple[str, ...]: ... @property def description(self) -> str | None: ... @@ -1668,6 +1672,8 @@ class NativeCompiledFunction: @property def packages(self) -> tuple[str, ...]: ... @property + def dataframe(self) -> str | None: ... + @property def tags(self) -> tuple[str, ...]: ... @property def description(self) -> str | None: ... @@ -9615,6 +9621,7 @@ class CompiledFunction(NativeRecord): runtime_version: str | None entry_point: str | None packages: tuple[str, ...] + dataframe: str | None tags: tuple[str, ...] description: str | None def __init__( @@ -9635,6 +9642,7 @@ class CompiledFunction(NativeRecord): runtime_version: str | None = ..., entry_point: str | None = ..., packages: tuple[str, ...] = ..., + dataframe: str | None = ..., tags: tuple[str, ...] = ..., description: str | None = ..., ) -> None: ... @@ -10060,7 +10068,7 @@ class DiscoveredModelSchemaFile(NativeRecord): ) -> None: ... class DiscoveredPythonFunctionFile(NativeRecord): - """A discovered Python function file and its parsed UDF metadata.""" + """A discovered Python function file and its parsed UDF or table function metadata.""" file_path: Path relative_path: Path @@ -10068,6 +10076,8 @@ class DiscoveredPythonFunctionFile(NativeRecord): header_values: dict[str, object] entry_point: str body_python: str + table_function: bool + dataframe: str | None def __init__( self, file_path: Path, @@ -10076,6 +10086,8 @@ class DiscoveredPythonFunctionFile(NativeRecord): header_values: dict[str, object], entry_point: str, body_python: str, + table_function: bool = ..., + dataframe: str | None = ..., ) -> None: ... class DiscoveredSchemaFile(NativeRecord): @@ -12271,6 +12283,7 @@ class DefaultsConfig(NativeRecord): seed_schema: str | None function_database: str | None function_schema: str | None + function_dataframe: str incremental_strategy: str | None incremental_mode: str | None microbatch_strategy: str | None @@ -12308,6 +12321,7 @@ class DefaultsConfig(NativeRecord): seed_schema: str | None = ..., function_database: str | None = ..., function_schema: str | None = ..., + function_dataframe: str = ..., incremental_strategy: str | None = ..., incremental_mode: str | None = ..., microbatch_strategy: str | None = ..., @@ -14325,6 +14339,7 @@ class FunctionPlanEntry(NativeRecord): runtime_version: str | None entry_point: str | None packages: tuple[str, ...] + dataframe: str | None previous_query_sql: str | None reason: PlanReason def __init__( @@ -14344,6 +14359,7 @@ class FunctionPlanEntry(NativeRecord): runtime_version: str | None = ..., entry_point: str | None = ..., packages: tuple[str, ...] = ..., + dataframe: str | None = ..., previous_query_sql: str | None = ..., reason: PlanReason = ..., ) -> None: ... @@ -15537,12 +15553,14 @@ class CompileSqlReference(NativeRecord): ref_name: str ref_package: str | None call_argument_count: int | None + call_partitioned: bool def __init__( self, ref_kind: SqlReferenceKind | str, ref_name: str, ref_package: str | None = ..., call_argument_count: int | None = ..., + call_partitioned: bool = ..., ) -> None: ... class CompiledLineageSourceFact(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 143a9e7c76..5144aa7a6e 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -658,8 +658,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: - del runtime_version, entry_point, packages + del runtime_version, entry_point, packages, dataframe if return_columns: raise AdapterUserError( message=f"Adapter '{type(self).__name__}' does not support SQL table functions" @@ -1283,6 +1284,7 @@ def create_function( runtime_version=definition.runtime_version, entry_point=definition.entry_point, packages=definition.packages, + dataframe=definition.dataframe, ) statement_recorder.record_many(statements) stmt: str diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 6929cd572c..42e2bc688f 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1422,8 +1422,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: - del runtime_version, entry_point, packages + del runtime_version, entry_point, packages, dataframe if language == FunctionLanguage.PYTHON: if return_columns: raise AdapterUserError(message="DuckDB table functions must use SQL language") diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 6b3f4d5b98..4e1f089a25 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -421,6 +421,7 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: """Render SQL statements that create or replace a SQL function.""" ... diff --git a/src/sqlbuild/adapter/contract/main/temporary_function_copy.py b/src/sqlbuild/adapter/contract/main/temporary_function_copy.py new file mode 100644 index 0000000000..7b8d365082 --- /dev/null +++ b/src/sqlbuild/adapter/contract/main/temporary_function_copy.py @@ -0,0 +1,19 @@ +"""Shared session TEMPORARY copies of a function's CREATE statements.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import TemporaryFunction + + +def temporary_function_copy(*, target: str, statements: tuple[str, ...]) -> TemporaryFunction: + """The `CREATE OR REPLACE FUNCTION` statements of `target` as TEMPORARY functions.""" + + return TemporaryFunction( + target=target, + statements=tuple( + statement.replace( + "CREATE OR REPLACE FUNCTION ", "CREATE OR REPLACE TEMPORARY FUNCTION ", 1 + ) + for statement in statements + ), + ) diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 33862792c6..72cc14ac34 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -317,6 +317,7 @@ class FunctionDefinition: runtime_version: str | None = None entry_point: str | None = None packages: tuple[str, ...] = () + dataframe: str | None = None source_file_path: Path | None = None diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index dbf477190b..90fc9df457 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -1766,7 +1766,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: + del dataframe argument_sql: str = ", ".join(f"{argument.name} {argument.type}" for argument in arguments) if return_columns: if language != FunctionLanguage.SQL: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 5f71843f45..321580ceff 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -930,10 +930,11 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: """Create a server-global SQL lambda named ``__``.""" - del runtime_version, entry_point, packages + del runtime_version, entry_point, packages, dataframe if language != FunctionLanguage.SQL: raise AdapterUserError( message="ClickHouse does not deploy Python functions; executable UDFs need " diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index a137aeaf7c..c3d55d6a3c 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -57,6 +57,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.temporary_function_copy import temporary_function_copy from sqlbuild.adapter.contract.models import ( ColumnInfo, CursorValue, @@ -1648,7 +1649,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: + del dataframe if language == FunctionLanguage.PYTHON: if return_columns: raise AdapterUserError(message="Databricks table functions must use SQL language") @@ -1691,15 +1694,7 @@ def render_create_temporary_function( body_sql=definition.body_sql, return_columns=definition.return_columns, ) - return TemporaryFunction( - target=target, - statements=tuple( - statement.replace( - "CREATE OR REPLACE FUNCTION ", "CREATE OR REPLACE TEMPORARY FUNCTION ", 1 - ) - for statement in statements - ), - ) + return temporary_function_copy(target=target, statements=statements) def _render_create_python_function( self, diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 9b056b7636..431886f3f2 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -558,8 +558,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: - del runtime_version, entry_point, packages + del runtime_version, entry_point, packages, dataframe arg_sql: str = ", ".join(f"{arg.name} {arg.type}" for arg in arguments) if language != FunctionLanguage.SQL: raise AdapterUserError( diff --git a/src/sqlbuild/adapters/snowflake/classes/python_table_function_handler.py b/src/sqlbuild/adapters/snowflake/classes/python_table_function_handler.py new file mode 100644 index 0000000000..096b5f02b4 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/classes/python_table_function_handler.py @@ -0,0 +1,89 @@ +"""The generated partition handler of a Snowflake Python table function.""" + +from __future__ import annotations + +import re + +from sqlbuild.adapters.snowflake.constants import ( + PANDAS_PACKAGES, + POLARS_DATAFRAME, + POLARS_PACKAGES, + PYTHON_TABLE_FUNCTION_HANDLER, +) + +_PACKAGE_NAME: re.Pattern[str] = re.compile(r"^\s*([A-Za-z0-9_.\-]+)") + + +class PythonTableFunctionHandler: + """A vectorized handler that calls a user's entry point once per partition.""" + + def __init__( + self, + *, + body: str, + entry_point: str, + argument_names: tuple[str, ...], + column_names: tuple[str, ...], + dataframe: str, + ) -> None: + self.body: str = body + self.entry_point: str = entry_point + self.argument_names: tuple[str, ...] = argument_names + self.column_names: tuple[str, ...] = column_names + self.dataframe: str = dataframe + + def packages(self, *, declared: tuple[str, ...]) -> tuple[str, ...]: + """The declared packages, then each package the partition frame needs that none names.""" + + required: tuple[str, ...] = ( + POLARS_PACKAGES if self.dataframe == POLARS_DATAFRAME else PANDAS_PACKAGES + ) + named: set[str] = { + match.group(1).lower() + for package in declared + if (match := _PACKAGE_NAME.match(package)) is not None + } + return (*declared, *(package for package in required if package not in named)) + + def module(self) -> str: + """The user's module followed by the handler class.""" + + polars: bool = self.dataframe == POLARS_DATAFRAME + imports: list[str] = ["import pandas as _sqb_pandas"] + if polars: + imports.append("import polars as _sqb_polars") + imports.append("from _snowflake import vectorized as _sqb_vectorized") + frame: str = "_sqb_polars.from_pandas(_sqb_partition)" if polars else "_sqb_partition" + output: str = ( + "_sqb_result.select(_sqb_expected).to_pandas()" + if polars + else "_sqb_result[_sqb_expected]" + ) + lines: list[str] = [ + self.body, + "", + *imports, + "", + "", + f"class {PYTHON_TABLE_FUNCTION_HANDLER}:", + " @_sqb_vectorized(input=_sqb_pandas.DataFrame)", + " def end_partition(self, _sqb_partition):", + f" _sqb_partition.columns = {list(self.argument_names)!r}", + f" _sqb_result = {self.entry_point}({frame})", + " _sqb_returned = list(_sqb_result.columns)", + f" _sqb_expected = {list(self.column_names)!r}", + " _sqb_missing = [name for name in _sqb_expected if name not in _sqb_returned]", + " _sqb_extra = [name for name in _sqb_returned if name not in _sqb_expected]", + " _sqb_repeated = sorted(", + " {name for name in _sqb_returned if _sqb_returned.count(name) > 1}", + " )", + " if _sqb_missing or _sqb_extra or _sqb_repeated:", + " raise ValueError(", + f' f"table function {self.entry_point} returned columns "', + ' f"{_sqb_returned}, but its "', + ' f"returns declare {_sqb_expected}: missing {_sqb_missing}, "', + ' f"extra {_sqb_extra}, repeated {_sqb_repeated}"', + " )", + f" return {output}", + ] + return "\n".join(lines) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 5d4f757b97..4737fbf828 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -61,6 +61,7 @@ from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.main.temporary_function_copy import temporary_function_copy from sqlbuild.adapter.contract.models import ( ColumnInfo, ExpressionInferenceProfile, @@ -158,6 +159,9 @@ from sqlbuild.adapters.snowflake._helpers.state_table_retention import ( is_invalid_retention_error, ) +from sqlbuild.adapters.snowflake.classes.python_table_function_handler import ( + PythonTableFunctionHandler, +) from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection from sqlbuild.adapters.snowflake.classes.state_table_retention_fallbacks import ( StateTableRetentionFallbacks, @@ -172,6 +176,7 @@ OAUTH_AUTHORIZATION_CODE_AUTHENTICATOR, PLATFORM_DETECTION_DISABLED, PLATFORM_DETECTION_VARIABLE, + PYTHON_TABLE_FUNCTION_HANDLER, SECONDARY_ROLES_ALL, SECONDARY_ROLES_NONE, SHOW_LIKE_ESCAPE_CHARACTER, @@ -1390,6 +1395,7 @@ def create_function( runtime_version=definition.runtime_version, entry_point=definition.entry_point, packages=definition.packages, + dataframe=definition.dataframe, ) statement_recorder.record_many(statements) stmt: str @@ -3180,18 +3186,43 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: argument_sql: str = ", ".join(f"{argument.name} {argument.type}" for argument in arguments) if return_columns: - if language != FunctionLanguage.SQL: - raise AdapterUserError(message="Snowflake table functions must use SQL language") column_sql: str = ", ".join(f"{column.name} {column.type}" for column in return_columns) - del returns, runtime_version, entry_point, packages + if language == FunctionLanguage.PYTHON: + if runtime_version is None or entry_point is None or dataframe is None: + raise AdapterUserError( + message=( + "Snowflake Python table functions require runtime_version, " + "entry_point and a dataframe library" + ) + ) + handler: PythonTableFunctionHandler = PythonTableFunctionHandler( + body=body_sql, + entry_point=entry_point, + argument_names=tuple(str(argument.name) for argument in arguments), + column_names=tuple(str(column.name) for column in return_columns), + dataframe=dataframe, + ) + table_packages: str = "','".join(handler.packages(declared=packages)) + return ( + f"CREATE OR REPLACE FUNCTION {destination}({argument_sql})\n" + f"RETURNS TABLE ({column_sql})\n" + "LANGUAGE PYTHON\n" + f"RUNTIME_VERSION = '{runtime_version}'\n" + f"HANDLER = '{PYTHON_TABLE_FUNCTION_HANDLER}'\n" + f"PACKAGES = ('{table_packages}')\n" + f"AS $$\n{handler.module()}\n$$", + ) + del returns, runtime_version, entry_point, packages, dataframe return ( f"CREATE OR REPLACE FUNCTION {destination}({argument_sql})\n" f"RETURNS TABLE ({column_sql})\n" f"AS $$\n{body_sql}\n$$", ) + del dataframe if language == FunctionLanguage.PYTHON: if runtime_version is None or entry_point is None: raise AdapterUserError( @@ -3233,16 +3264,9 @@ def render_create_temporary_function( runtime_version=definition.runtime_version, entry_point=definition.entry_point, packages=definition.packages, + dataframe=definition.dataframe, ) - return TemporaryFunction( - target=target, - statements=tuple( - statement.replace( - "CREATE OR REPLACE FUNCTION ", "CREATE OR REPLACE TEMPORARY FUNCTION ", 1 - ) - for statement in statements - ), - ) + return temporary_function_copy(target=target, statements=statements) def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: if cursor_type == CursorKind.INTEGER: diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 78b6e350a4..ffe893ff4d 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -50,3 +50,7 @@ FIXED_POINT_RESULT_TYPE: str = "FIXED" PLATFORM_DETECTION_VARIABLE: str = "SNOWFLAKE_DISABLE_PLATFORM_DETECTION" PLATFORM_DETECTION_DISABLED: str = "true" +POLARS_DATAFRAME: str = "polars" +PYTHON_TABLE_FUNCTION_HANDLER: str = "_SqbHandler" +PANDAS_PACKAGES: tuple[str, ...] = ("pandas",) +POLARS_PACKAGES: tuple[str, ...] = ("pandas", "polars", "pyarrow") diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 664fac4c02..478dd82d06 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -665,8 +665,18 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: - del arguments, returns, body_sql, return_columns, runtime_version, entry_point, packages + del ( + arguments, + returns, + body_sql, + return_columns, + runtime_version, + entry_point, + packages, + dataframe, + ) raise AdapterUserError( message=( f"SQLite cannot create function '{destination}': SQLite has no CREATE FUNCTION, " diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 199b7c7770..bc61ec7eb8 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -826,8 +826,9 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: - del return_columns, runtime_version, entry_point, packages + del return_columns, runtime_version, entry_point, packages, dataframe if language != FunctionLanguage.SQL: raise AdapterUserError( message="Python functions require an engine-specific implementation" diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 4915b9723d..66e9cc142a 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -146,6 +146,8 @@ def entry_point(self) -> str | None: ... @property def packages(self) -> tuple[str, ...]: ... @property + def dataframe(self) -> str | None: ... + @property def tags(self) -> tuple[str, ...]: ... @property def description(self) -> str | None: ... diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index c7f935e162..d2dc019bb7 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -86,6 +86,7 @@ def execute_function( runtime_version=function_entry.runtime_version, entry_point=function_entry.entry_point, packages=function_entry.packages, + dataframe=function_entry.dataframe, source_file_path=function_entry.source_file_path, ), statement_recorder=statement_recorder, diff --git a/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py b/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py index 734d328678..6d38cb6f87 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py +++ b/src/sqlbuild/executor/pipeline/_helpers/temporary_functions.py @@ -81,6 +81,7 @@ def create_temporary_test_functions( runtime_version=entry.runtime_version, entry_point=entry.entry_point, packages=entry.packages, + dataframe=entry.dataframe, source_file_path=entry.source_file_path, ), ) diff --git a/src/sqlbuild/functions.py b/src/sqlbuild/functions.py index 6b5f54b633..d45a3d926f 100644 --- a/src/sqlbuild/functions.py +++ b/src/sqlbuild/functions.py @@ -1,5 +1,6 @@ """Authoring helpers for SQLBuild-managed warehouse functions.""" +from sqlbuild.python_nodes.main.table_fn import table_fn as table_fn from sqlbuild.python_nodes.main.udf import udf as udf -__all__ = ("udf",) +__all__ = ("table_fn", "udf") diff --git a/src/sqlbuild/python_nodes/main/table_fn.py b/src/sqlbuild/python_nodes/main/table_fn.py new file mode 100644 index 0000000000..f4abd9a9ca --- /dev/null +++ b/src/sqlbuild/python_nodes/main/table_fn.py @@ -0,0 +1,17 @@ +"""Authoring helper for SQLBuild-managed Python table functions.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +def table_fn(**kwargs: object) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Mark a Python function as a SQLBuild table function for static project discovery.""" + + del kwargs + + def _decorate(function: Callable[..., Any]) -> Callable[..., Any]: + return function + + return _decorate diff --git a/tests/goldens/compiler/failure/config-unknown-default-key.json b/tests/goldens/compiler/failure/config-unknown-default-key.json index 9ad1c9ca74..486acadfc9 100644 --- a/tests/goldens/compiler/failure/config-unknown-default-key.json +++ b/tests/goldens/compiler/failure/config-unknown-default-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml [defaults] contains unknown key(s): description. Allowed keys: append_cursor_inclusive, batch_concurrency, batch_size, contract, cursor_end, cursor_future_action, cursor_future_max_distance, cursor_start, cursor_start_max_action, cursor_start_max_ahead, cursor_watermark_mode, database, full_refresh, function_database, function_schema, incremental_mode, incremental_strategy, lookback, materialized, max_duration, max_microbatches, merge_exclude_columns, microbatch_strategy, replay_on_change, row_diff_exclude_columns, row_diff_sample_rows, row_diff_sample_seed, row_diff_tolerances, schema, seed_database, seed_schema, tags, unaccounted_partition_policy", + "message": "/sqlbuild_project.toml [defaults] contains unknown key(s): description. Allowed keys: append_cursor_inclusive, batch_concurrency, batch_size, contract, cursor_end, cursor_future_action, cursor_future_max_distance, cursor_start, cursor_start_max_action, cursor_start_max_ahead, cursor_watermark_mode, database, full_refresh, function_database, function_dataframe, function_schema, incremental_mode, incremental_strategy, lookback, materialized, max_duration, max_microbatches, merge_exclude_columns, microbatch_strategy, replay_on_change, row_diff_exclude_columns, row_diff_sample_rows, row_diff_sample_seed, row_diff_tolerances, schema, seed_database, seed_schema, tags, unaccounted_partition_policy", "help": "remove the key; see the reference for supported keys" } ] diff --git a/tests/goldens/python_oracle/macro_expansion_seeded_fragments.json b/tests/goldens/python_oracle/macro_expansion_seeded_fragments.json index 9b53246420..68a8d300b4 100644 --- a/tests/goldens/python_oracle/macro_expansion_seeded_fragments.json +++ b/tests/goldens/python_oracle/macro_expansion_seeded_fragments.json @@ -125,7 +125,7 @@ ], [ "macro_expansion", -"d5267f01981f3e0cd620d1c5" +"1029b33758ce1113ba181aad" ], [ "macro_expansion", @@ -173,7 +173,7 @@ ], [ "macro_expansion", -"f1242803e3f38f2f1b6590da" +"0fe41ff531ce4ba37966ea82" ], [ "macro_expansion", @@ -209,7 +209,7 @@ ], [ "macro_expansion", -"a0d0c25362e730abb95e72ab" +"60c02ffa6ccd1a87bfd0aa94" ], [ "macro_expansion", @@ -229,7 +229,7 @@ ], [ "macro_expansion", -"831718808456579f69edfad4" +"4467128d6cd83c60c7354d0a" ], [ "macro_expansion", @@ -265,7 +265,7 @@ ], [ "macro_expansion", -"0a9401d157da584e1a07df85" +"a396dc5118bca8abbdb49bb2" ], [ "macro_expansion", @@ -293,7 +293,7 @@ ], [ "macro_expansion", -"3aed073a950cc4e154c0490b" +"51c4f04edf220481f1c58372" ], [ "macro_expansion", @@ -317,7 +317,7 @@ ], [ "macro_expansion", -"001e21afaa80f0e3d858fc84" +"fa1eef32f3d6f4254f638c4c" ], [ "macro_expansion", @@ -337,7 +337,7 @@ ], [ "macro_expansion", -"a5f1b81418b99c79dc858fc0" +"59a4300991801e24ca3c516f" ], [ "macro_expansion", @@ -345,7 +345,7 @@ ], [ "macro_expansion", -"a01a1ef95e287fd2ac2816f4" +"2eec7b4ab6495a9e8694dc6a" ], [ "macro_expansion", @@ -389,7 +389,7 @@ ], [ "macro_expansion", -"8df9f60723f05a17f6ded287" +"30e42a02218e0b2bce9843c5" ], [ "macro_expansion", @@ -501,7 +501,7 @@ ], [ "macro_expansion", -"44ec1529c59a7531902f266a" +"c9ef8ca8270da0c96152e6ae" ], [ "macro_expansion", @@ -537,7 +537,7 @@ ], [ "macro_expansion", -"a2dce238490c31e89d5bd6be" +"b190d30a3cecda9bffffbf1a" ], [ "macro_expansion", @@ -613,7 +613,7 @@ ], [ "macro_expansion", -"657138655faac024028d93e2" +"5b9634357fa03154a9086381" ], [ "macro_expansion", @@ -725,11 +725,11 @@ ], [ "macro_expansion", -"cbcbd483eed139a1a234bfd3" +"525c5ed1624ef841736e4ea5" ], [ "macro_expansion", -"643a2a7c2e9dc6e97e0617f8" +"926f14a65b6001c2db30b485" ], [ "macro_expansion", @@ -809,7 +809,7 @@ ], [ "macro_expansion", -"bea1ffb1af856ccd4f5f32a3" +"21c39c6bfe341d114ddbdc83" ], [ "macro_expansion", @@ -909,7 +909,7 @@ ], [ "macro_expansion", -"32d698ab1460d1c80a0e8231" +"7312c4d9fec11263318af57f" ], [ "macro_expansion", @@ -941,11 +941,11 @@ ], [ "macro_expansion", -"95e6fdbb0efd9309bed4186c" +"ea21e8733299d0bde326891a" ], [ "macro_expansion", -"de9a42488eaba3a128ee0434" +"29cefe90a1c4f854446dc0df" ], [ "macro_expansion", @@ -961,7 +961,7 @@ ], [ "macro_expansion", -"36966a8979984ac21cfb15c8" +"8116e58fa062e275d08a4c7e" ], [ "macro_expansion", @@ -1025,7 +1025,7 @@ ], [ "macro_expansion", -"13cfecd165d77b7534f6444d" +"972086d46bc9dcd606c2eb22" ], [ "macro_expansion", @@ -1097,7 +1097,7 @@ ], [ "macro_expansion", -"3c64db8bb2d0632a6f1270a7" +"2053f43db270a93d09b28369" ], [ "macro_expansion", @@ -1173,7 +1173,7 @@ ], [ "macro_expansion", -"3ec33f040f4b9c28151196a6" +"7b9d4ba01eef6c05a6bcae49" ], [ "macro_expansion", @@ -1241,7 +1241,7 @@ ], [ "macro_expansion", -"555759d41c5c6cbc5a87149a" +"c9f12c1de0e883a718967548" ], [ "macro_expansion", @@ -1277,7 +1277,7 @@ ], [ "macro_expansion", -"b0ec0cbc9fb5ee454a3250ad" +"29e36fab22d872f10de7eb32" ], [ "macro_expansion", @@ -1421,7 +1421,7 @@ ], [ "macro_expansion", -"ab47d81c3395fe0ca863543f" +"599e821ee97285b782ee86b2" ], [ "macro_expansion", @@ -1505,7 +1505,7 @@ ], [ "macro_expansion", -"9b2760de2123bde239c09bcc" +"0a03dfbd1572614bf33f4570" ], [ "macro_expansion", @@ -1533,11 +1533,11 @@ ], [ "macro_expansion", -"3c2e40077c9d2dfe089d3a04" +"cb79840e0a69f44c5d4c3956" ], [ "macro_expansion", -"cc18b5c5fb8b22c45e3e3c6b" +"1fc6ecef4a3b162105f44d31" ], [ "macro_expansion", @@ -1601,7 +1601,7 @@ ], [ "macro_expansion", -"06cc41fc2e004b0bf0a7c54b" +"2bd9681c3e9a5cc377cfe943" ], [ "macro_expansion", @@ -1669,15 +1669,15 @@ ], [ "macro_expansion", -"1bf3b929f0391485fc59e881" +"9ee42dd9bb5eb85151aec63f" ], [ "macro_expansion", -"9884dcf27956ae8dc2c02ba3" +"89359f560da82bd5b53f0947" ], [ "macro_expansion", -"e166954da7b149220934be76" +"3656e1cee21069b5eb779753" ], [ "macro_expansion", @@ -1729,7 +1729,7 @@ ], [ "macro_expansion", -"138c2e17e1fc2d5ca491567f" +"1fb0a65b2851dd0ed77bf3e9" ], [ "macro_expansion", @@ -1909,7 +1909,7 @@ ], [ "macro_expansion", -"4787957c4eef4cff14d0377c" +"67e1f0b0749acacc059a2c76" ], [ "macro_expansion", @@ -1977,7 +1977,7 @@ ], [ "macro_expansion", -"0d337b829634a30bbb05b69f" +"fc3f12aef486f3d4e6ed3fb4" ], [ "macro_expansion", @@ -1997,11 +1997,11 @@ ], [ "macro_expansion", -"7f802c6210eb10aa2f596e1c" +"ce2659269d8de17033e067b5" ], [ "macro_expansion", -"faba2cda28f5e35dc2608bce" +"ba19359265de0a78c35da448" ], [ "macro_expansion", @@ -2037,7 +2037,7 @@ ], [ "macro_expansion", -"b6911b261df3f6a092671423" +"46adf1d201e4ec8d3499fcf1" ], [ "macro_expansion", @@ -2113,7 +2113,7 @@ ], [ "macro_expansion", -"632c69bd022be6d479c34728" +"8e489d7beeec6507f6d9b377" ], [ "macro_expansion", @@ -2129,7 +2129,7 @@ ], [ "macro_expansion", -"a175535bab9f848b6eac9d0f" +"35f5533cf90743232f5514dc" ], [ "macro_expansion", @@ -2289,11 +2289,11 @@ ], [ "macro_expansion", -"c2a78c310fcf60d1235ac03e" +"abac4912604be90c82e9daf4" ], [ "macro_expansion", -"2ef39152757336dfbb2584c8" +"3ba98d927d5f87377cf282d1" ], [ "macro_expansion", @@ -2309,7 +2309,7 @@ ], [ "macro_expansion", -"fe74ede5a6d3cb03a5fd8acc" +"a2c7b9ec1608f038fbacd2b2" ], [ "macro_expansion", @@ -2317,7 +2317,7 @@ ], [ "macro_expansion", -"d68a16c4c6d552539a4cc0aa" +"2b2b765c87db75bb16e780bb" ], [ "macro_expansion", @@ -2333,7 +2333,7 @@ ], [ "macro_expansion", -"4d53e449d281fcf8fb6f5809" +"151d93cfd558bf365ec9b58b" ], [ "macro_expansion", @@ -2345,7 +2345,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -2401,7 +2401,7 @@ ], [ "macro_expansion", -"7f390d152c7f1de7be4f46c8" +"949765572c06b3cd792a02f2" ], [ "macro_expansion", @@ -2409,7 +2409,7 @@ ], [ "macro_expansion", -"38bb1d81f169c54cb6422e07" +"e1ac89d053c96e78f8b3fdac" ], [ "macro_expansion", @@ -2589,11 +2589,11 @@ ], [ "macro_expansion", -"652999c5201b8a0189f04255" +"bbc1365d4fdb2bde052ef997" ], [ "macro_expansion", -"9c4e095f9c6721c030bafb1f" +"8af9c33809a8fc3564e1d5b0" ], [ "macro_expansion", @@ -2605,7 +2605,7 @@ ], [ "macro_expansion", -"61e532def5768b50918dd1b6" +"672162eefb6f6e5e6962f280" ], [ "macro_expansion", @@ -2629,7 +2629,7 @@ ], [ "macro_expansion", -"7fbe89a74922f47119cda771" +"48f6bfffd7c4226135936e2e" ], [ "macro_expansion", @@ -2725,7 +2725,7 @@ ], [ "macro_expansion", -"9e5f00417041e493606b2cb6" +"e91f75fc3cec95a6d427fba8" ], [ "macro_expansion", @@ -2737,7 +2737,7 @@ ], [ "macro_expansion", -"cf5c8ff68b780738b8d6332d" +"8d4732bbebcbf81b153c9f9e" ], [ "macro_expansion", @@ -2789,7 +2789,7 @@ ], [ "macro_expansion", -"e5aa558e30465a2f100b0d72" +"933c59b48aac040d608debd1" ], [ "macro_expansion", @@ -2805,7 +2805,7 @@ ], [ "macro_expansion", -"91009fcab331d25580ec1509" +"1fff6f675a5359057091edd2" ], [ "macro_expansion", @@ -2813,7 +2813,7 @@ ], [ "macro_expansion", -"c99398a1eb56cd28d1a0fc5f" +"23ac6f1e4387a148f776ca3e" ], [ "macro_expansion", @@ -2829,11 +2829,11 @@ ], [ "macro_expansion", -"e8f40a915edf1055e478dd7c" +"7bb44850758b4463bf0a50a6" ], [ "macro_expansion", -"32a58085d1a7fd541c6a88f1" +"445d7514bb230cacebe018ab" ], [ "macro_expansion", @@ -2893,7 +2893,7 @@ ], [ "macro_expansion", -"3083b54b68377afa9d534e29" +"dc91caa627dc060a476ea2e6" ], [ "macro_expansion", @@ -2933,7 +2933,7 @@ ], [ "macro_expansion", -"f961986a7a8d381b10c97e80" +"81fa58cd00aa1ef3bf1cebc3" ], [ "macro_expansion", @@ -2941,7 +2941,7 @@ ], [ "macro_expansion", -"4968827a25d4c75b2513147f" +"546953a416edb5b9b64abbde" ], [ "macro_expansion", @@ -3033,7 +3033,7 @@ ], [ "macro_expansion", -"58a12d40952f3d25037196b4" +"756839047a85854a91a85b84" ], [ "macro_expansion", @@ -3065,7 +3065,7 @@ ], [ "macro_expansion", -"5c04293fd803516b29f8955f" +"5ef780fc0e56e19ee7d0cb1e" ], [ "macro_expansion", @@ -3077,7 +3077,7 @@ ], [ "macro_expansion", -"cf853c5b491953123dc87898" +"79025db9f8ea2f6dda4e5270" ], [ "macro_expansion", @@ -3093,11 +3093,11 @@ ], [ "macro_expansion", -"e34b5f16f2bfbf18948f1424" +"832f4cb629d4462b6d62a4b1" ], [ "macro_expansion", -"a9e59161d994c35e45d1976d" +"e27e96a7e2cc0737048b5151" ], [ "macro_expansion", @@ -3113,7 +3113,7 @@ ], [ "macro_expansion", -"079a6b40444a218f944ec8ca" +"7fa610666d9b0f923f881dfb" ], [ "macro_expansion", @@ -3177,7 +3177,7 @@ ], [ "macro_expansion", -"adbbbf8326cffa55781fb793" +"b9243c7537575787a69880fd" ], [ "macro_expansion", @@ -3197,7 +3197,7 @@ ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -3257,7 +3257,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -3293,7 +3293,7 @@ ], [ "macro_expansion", -"81c33d1c7fed38212a060ac4" +"96a60e34c899c1ba712c4381" ], [ "macro_expansion", @@ -3329,7 +3329,7 @@ ], [ "macro_expansion", -"a58b2cf112529be204819524" +"792aef79409b9c79adbc48e5" ], [ "macro_expansion", @@ -3345,7 +3345,7 @@ ], [ "macro_expansion", -"7823f6185f263760f041f9a0" +"397ea0953c8fb74cd9089719" ], [ "macro_expansion", @@ -3393,11 +3393,11 @@ ], [ "macro_expansion", -"91a080f61e30cac0207ce28e" +"34e20d67cbbca7a665045888" ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -3409,7 +3409,7 @@ ], [ "macro_expansion", -"f4d0383b6aae6a4679958598" +"4f3b6fab869543cbe5846d70" ], [ "macro_expansion", @@ -3417,7 +3417,7 @@ ], [ "macro_expansion", -"6e98435414d54fe523897353" +"83c49b17c010b1cb0689a8f0" ], [ "macro_expansion", @@ -3425,7 +3425,7 @@ ], [ "macro_expansion", -"6473f23c12676cf480a30933" +"c8b44800606bf9a4ea8ce0e2" ], [ "macro_expansion", @@ -3433,7 +3433,7 @@ ], [ "macro_expansion", -"aa6b76f363b3380a692c33d5" +"51b6a41f74b5f5b883ea25f5" ], [ "macro_expansion", @@ -3485,7 +3485,7 @@ ], [ "macro_expansion", -"24bba17bbe49fb072ef5f3e3" +"c93a3b53ea09f2c976054457" ], [ "macro_expansion", @@ -3533,7 +3533,7 @@ ], [ "macro_expansion", -"5de43e08e886c8d317c5555a" +"1a6bbf1fa0bafad2e49f307f" ], [ "macro_expansion", @@ -3565,7 +3565,7 @@ ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -3721,7 +3721,7 @@ ], [ "macro_expansion", -"80ecba4a63ec8efb0709cad4" +"e77cbe6f87ec73cd6f5b409c" ], [ "macro_expansion", @@ -3737,7 +3737,7 @@ ], [ "macro_expansion", -"7f802c6210eb10aa2f596e1c" +"ce2659269d8de17033e067b5" ], [ "macro_expansion", @@ -3777,7 +3777,7 @@ ], [ "macro_expansion", -"d2608491aae3389db4541669" +"ea8182ef2a69ee5c0d449363" ], [ "macro_expansion", @@ -3793,11 +3793,11 @@ ], [ "macro_expansion", -"7a5c180bb0d5277fdda04d89" +"a349d371664c3066d69fee65" ], [ "macro_expansion", -"3b7820fba55c362348dd709b" +"298fec77d01cf3fcbb1fdccb" ], [ "macro_expansion", @@ -3821,7 +3821,7 @@ ], [ "macro_expansion", -"62ec6e0d9994a1decbaca696" +"72a46d55f052a23a3a98808e" ], [ "macro_expansion", @@ -3905,7 +3905,7 @@ ], [ "macro_expansion", -"5fcdf53d3d00cc42bcbf7a51" +"9a6d841211a26ad5e4113451" ], [ "macro_expansion", @@ -4073,11 +4073,11 @@ ], [ "macro_expansion", -"47ec4f20baf7b205efa9e353" +"0abe642894b99128754fea6d" ], [ "macro_expansion", -"ef0b7eae2e7b0832da974e88" +"eee59e14cae0e274fd03e229" ], [ "macro_expansion", @@ -4129,7 +4129,7 @@ ], [ "macro_expansion", -"87a13e45065f01caaff61f07" +"2ee5c9f1d6a44f518bf2c4e0" ], [ "macro_expansion", @@ -4137,7 +4137,7 @@ ], [ "macro_expansion", -"742db5b2bf9468ef4aea7666" +"e6c23cf76d691ef29bf04bc8" ], [ "macro_expansion", @@ -4165,7 +4165,7 @@ ], [ "macro_expansion", -"f1f2a1738eb0a9a62b925857" +"1e13983084082569d42441fb" ], [ "macro_expansion", @@ -4173,7 +4173,7 @@ ], [ "macro_expansion", -"deb837e471a85bacfa80cd54" +"ff8b657223e001f06f648930" ], [ "macro_expansion", @@ -4365,7 +4365,7 @@ ], [ "macro_expansion", -"1d118dbc74a8e533ee0ce879" +"c9d801e4d0f285b0b26b46ee" ], [ "macro_expansion", @@ -4385,7 +4385,7 @@ ], [ "macro_expansion", -"b11a8d0d0c6b2a1d663eccf8" +"d7252f7f0813f50e343e2bae" ], [ "macro_expansion", @@ -4429,7 +4429,7 @@ ], [ "macro_expansion", -"ac056bb9660ceab859501164" +"393d368f80599b3ee16c9c22" ], [ "macro_expansion", @@ -4449,7 +4449,7 @@ ], [ "macro_expansion", -"3d9b19b4544bc1ac450015ad" +"735083f840d7117244dc4bdb" ], [ "macro_expansion", @@ -4457,7 +4457,7 @@ ], [ "macro_expansion", -"f2c35b6d8caa14eea1758279" +"8874b3c72b63ee7ed94fbfe8" ], [ "macro_expansion", @@ -4477,15 +4477,15 @@ ], [ "macro_expansion", -"60e522b81869b512ef367b8f" +"de9692f870ec3c5c2430d7c2" ], [ "macro_expansion", -"68f074cd0aa2816a2e0520cc" +"36deb7c21997acbf07d833f6" ], [ "macro_expansion", -"4cce40a70e3d983257751cbd" +"c7d28d0ae0bfccb092b4e6c2" ], [ "macro_expansion", @@ -4529,7 +4529,7 @@ ], [ "macro_expansion", -"db16ddb88ff46cec0eebd843" +"c7cba704890939eba8dd1df6" ], [ "macro_expansion", @@ -4561,7 +4561,7 @@ ], [ "macro_expansion", -"3df7a7069cf9163d405da435" +"7afe56764782a37348031ae7" ], [ "macro_expansion", @@ -4597,7 +4597,7 @@ ], [ "macro_expansion", -"8127060bd8aca157e11cfd9a" +"9bd1c05622b3b46956506e78" ], [ "macro_expansion", @@ -4701,7 +4701,7 @@ ], [ "macro_expansion", -"03b87799fe82f9793bfb932b" +"8c34afe02d5c0906a908cf92" ], [ "macro_expansion", @@ -4797,7 +4797,7 @@ ], [ "macro_expansion", -"913f4f88596d0be6b192cbbb" +"18e24fcfa056eda3722e5032" ], [ "macro_expansion", @@ -4809,7 +4809,7 @@ ], [ "macro_expansion", -"24efefa95675a8446a941fc1" +"d18ed923c756395ec08c1877" ], [ "macro_expansion", @@ -4849,7 +4849,7 @@ ], [ "macro_expansion", -"a5ad3620d64ce4cd532c564e" +"51b534a7fb831ab29efdfaff" ], [ "macro_expansion", @@ -4901,7 +4901,7 @@ ], [ "macro_expansion", -"cb42d167f1cc28c41d509b9f" +"17fdd06455d68303f5c6bb50" ], [ "macro_expansion", @@ -4909,7 +4909,7 @@ ], [ "macro_expansion", -"b29d69a4cdb29da762968435" +"2945b3d37d1e6c40e58099d2" ], [ "macro_expansion", @@ -4921,7 +4921,7 @@ ], [ "macro_expansion", -"1a6d056058e7edc33bda4033" +"f5a2f7ea78b4b3e72c307534" ], [ "macro_expansion", @@ -4941,7 +4941,7 @@ ], [ "macro_expansion", -"8c57ae56941e5a6580b02eb8" +"c2f8795f79495a3526736910" ], [ "macro_expansion", @@ -4961,7 +4961,7 @@ ], [ "macro_expansion", -"410cb27acc5766d4cbd43f6a" +"8a9cac02da3991d50487df18" ], [ "macro_expansion", @@ -4973,7 +4973,7 @@ ], [ "macro_expansion", -"57a6ff63a32db742c313b584" +"7da36d08823d0408ef87bd43" ], [ "macro_expansion", @@ -5001,7 +5001,7 @@ ], [ "macro_expansion", -"66eadcddc408d32a97e4b557" +"6ed24c88b2ccfce21935d8c5" ], [ "macro_expansion", @@ -5085,7 +5085,7 @@ ], [ "macro_expansion", -"65ad672f2739d553fc7a70ad" +"d3f386cfde668f77516b103e" ], [ "macro_expansion", @@ -5149,11 +5149,11 @@ ], [ "macro_expansion", -"1df5e1c61de9856a46aab9a1" +"83eee5a8e0f811b336404414" ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -5213,7 +5213,7 @@ ], [ "macro_expansion", -"5d2a698624aae0bfac081be5" +"e7d5445905843ad6068df547" ], [ "macro_expansion", @@ -5273,7 +5273,7 @@ ], [ "macro_expansion", -"19e76f03a67dc38faac00aab" +"0772195fcfe8d38b399fafd6" ], [ "macro_expansion", @@ -5325,7 +5325,7 @@ ], [ "macro_expansion", -"6f6beab9bf01caf6bb55ed30" +"e6b30bb7a59afd7194a67fcc" ], [ "macro_expansion", @@ -5361,7 +5361,7 @@ ], [ "macro_expansion", -"1ffd84c237de89b753972b24" +"5930466ceeb5a4dd4a80b158" ], [ "macro_expansion", @@ -5393,7 +5393,7 @@ ], [ "macro_expansion", -"8e62893ffd6ef86cf578da17" +"e003061e2ac81cdb5b1298cd" ], [ "macro_expansion", @@ -5409,7 +5409,7 @@ ], [ "macro_expansion", -"62479df84bf9263cdc05a2d6" +"c21e61b310116e9514bec50c" ], [ "macro_expansion", @@ -5437,7 +5437,7 @@ ], [ "macro_expansion", -"6d6585907583efc2570531e3" +"4fdb4ffc9a389c5f71b50f9a" ], [ "macro_expansion", @@ -5497,7 +5497,7 @@ ], [ "macro_expansion", -"0a0f0e7a5a790087e9496ed6" +"11ded37839065db6f6eda607" ], [ "macro_expansion", @@ -5517,7 +5517,7 @@ ], [ "macro_expansion", -"6aa2cb9959e3b6be97686cbc" +"f9e86c03c654d784edb69c36" ], [ "macro_expansion", @@ -5529,7 +5529,7 @@ ], [ "macro_expansion", -"1d4b5cd48bf27f9d00eaa41f" +"f7d36fdf121cbcb99b4b4aa5" ], [ "macro_expansion", @@ -5565,7 +5565,7 @@ ], [ "macro_expansion", -"bda53449de265273e6249ad3" +"7a92cf3ef73d072c33b305df" ], [ "macro_expansion", @@ -5629,7 +5629,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -5689,7 +5689,7 @@ ], [ "macro_expansion", -"c88aae4c098ba3eca98f1757" +"28f2bdfb0b99e2ac450d8d7b" ], [ "macro_expansion", @@ -5797,7 +5797,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -5817,7 +5817,7 @@ ], [ "macro_expansion", -"aff55b3e421ee2c7ada2b232" +"788f417125e899718b328704" ], [ "macro_expansion", @@ -5965,7 +5965,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -5973,7 +5973,7 @@ ], [ "macro_expansion", -"b81734183a07e77dbc8b1e3e" +"38f728da527c3f050b6ad0dd" ], [ "macro_expansion", @@ -5981,7 +5981,7 @@ ], [ "macro_expansion", -"4787957c4eef4cff14d0377c" +"67e1f0b0749acacc059a2c76" ], [ "macro_expansion", @@ -6005,7 +6005,7 @@ ], [ "macro_expansion", -"557dddbe2b8ef89d24471456" +"1af3dc75bb0fbe754aecc581" ], [ "macro_expansion", @@ -6093,7 +6093,7 @@ ], [ "macro_expansion", -"6a0d7a266705600f099f980d" +"eac63673e1aabd716e05b572" ], [ "macro_expansion", @@ -6449,11 +6449,11 @@ ], [ "macro_expansion", -"5dcbb7543eb626b79e1ab3a4" +"ff80167da8acda473799e5f3" ], [ "macro_expansion", -"fba6f4e7131f8634442ee596" +"306364ca11026bd3cbf7e423" ], [ "macro_expansion", @@ -6473,7 +6473,7 @@ ], [ "macro_expansion", -"5afe186b309a9586373cb816" +"e53507e1526acc52a0779bce" ], [ "macro_expansion", @@ -6513,7 +6513,7 @@ ], [ "macro_expansion", -"17fadc5bc0b901390171f529" +"ae68dc477584331120a6e23a" ], [ "macro_expansion", @@ -6549,7 +6549,7 @@ ], [ "macro_expansion", -"0ada0d278c74ca5f11754277" +"ca4d694d8ea4223abc1bfa04" ], [ "macro_expansion", @@ -6561,7 +6561,7 @@ ], [ "macro_expansion", -"f4fb8736e91b2c8a09f62b00" +"c3bfd3042bf3637e13e61392" ], [ "macro_expansion", @@ -6573,7 +6573,7 @@ ], [ "macro_expansion", -"b89a1832b601987d8607e51d" +"ccddd82f5810cd533ee7fb21" ], [ "macro_expansion", @@ -6625,7 +6625,7 @@ ], [ "macro_expansion", -"37c4d9ec23989ec0a783c41f" +"7f6a19c6dc2c97015e70da09" ], [ "macro_expansion", @@ -6641,11 +6641,11 @@ ], [ "macro_expansion", -"1e4a6e8e080f72c04473774e" +"efd2600a55361e1aea4fb115" ], [ "macro_expansion", -"301545dbd64e0816c2bde178" +"a4e49ac65c43553e8c76f0ac" ], [ "macro_expansion", @@ -6665,7 +6665,7 @@ ], [ "macro_expansion", -"d4874f3a779adf4e6ad8cadd" +"b4acf20cafec87f3deb66f71" ], [ "macro_expansion", @@ -6693,7 +6693,7 @@ ], [ "macro_expansion", -"82417f5cca8e1f03b2da845d" +"b889599f3f6e5dca38e79d86" ], [ "macro_expansion", @@ -6753,7 +6753,7 @@ ], [ "macro_expansion", -"a91965b16db60f0ea8d2e3a0" +"43fe1392dbede70ec4b70f56" ], [ "macro_expansion", @@ -6773,7 +6773,7 @@ ], [ "macro_expansion", -"7f802c6210eb10aa2f596e1c" +"ce2659269d8de17033e067b5" ], [ "macro_expansion", @@ -6797,7 +6797,7 @@ ], [ "macro_expansion", -"96f93c2f4b773fc6c0f7f59d" +"cdb01ba1d6a37ecf5716ee02" ], [ "macro_expansion", @@ -6925,7 +6925,7 @@ ], [ "macro_expansion", -"eef8185a838b6503a18c829a" +"1aef0f180ca5cea476ce76be" ], [ "macro_expansion", @@ -6953,7 +6953,7 @@ ], [ "macro_expansion", -"5a4046fb8d7ca915e90dbdfb" +"3ca3febe7b187e580074ceb8" ], [ "macro_expansion", @@ -6981,7 +6981,7 @@ ], [ "macro_expansion", -"a1083dac226521f66db108e3" +"298db1a5c8f3f8b5b8fe53c6" ], [ "macro_expansion", @@ -7017,7 +7017,7 @@ ], [ "macro_expansion", -"53f234c92669230f8c720e6f" +"d0ec96a62704cef2870ff8f9" ], [ "macro_expansion", @@ -7057,11 +7057,11 @@ ], [ "macro_expansion", -"1ee3444ad7ae7ef61dff920b" +"ef838bbb97036faaf143432b" ], [ "macro_expansion", -"15b975faeac5f7a827f0d5d0" +"c66b71b4d30eaa596e2899f9" ], [ "macro_expansion", @@ -7105,7 +7105,7 @@ ], [ "macro_expansion", -"c4adfbe6db70cabb81d30dc6" +"75ea4dd70ed76e935ed3556a" ], [ "macro_expansion", @@ -7161,7 +7161,7 @@ ], [ "macro_expansion", -"2ab561f366de1abec18c156c" +"c2f390060fca147cfdac79ee" ], [ "macro_expansion", @@ -7185,7 +7185,7 @@ ], [ "macro_expansion", -"7819158d003b4dbab48716a4" +"0ad3aa69612e86c3110a801b" ], [ "macro_expansion", @@ -7193,7 +7193,7 @@ ], [ "macro_expansion", -"109cf6636b1cd78e899b33cf" +"5308017786e4da5cbe788713" ], [ "macro_expansion", @@ -7229,11 +7229,11 @@ ], [ "macro_expansion", -"3a45b11ac4f5beddeec06fdd" +"774410ead03931ec0d30247d" ], [ "macro_expansion", -"8bfb50e433b595a25b256f16" +"8cc046406da960baf3b686c8" ], [ "macro_expansion", @@ -7341,7 +7341,7 @@ ], [ "macro_expansion", -"cb1e82b71b075e7d4a1704da" +"04700c65e04ff7336ad4d939" ], [ "macro_expansion", @@ -7417,7 +7417,7 @@ ], [ "macro_expansion", -"6530dbf48ce7e5186c273021" +"a64b920fe2bbd6c59f1f4519" ], [ "macro_expansion", @@ -7477,7 +7477,7 @@ ], [ "macro_expansion", -"53436244a4d244e9c310b7a9" +"b669bc5622371f62f4e69c05" ], [ "macro_expansion", @@ -7553,7 +7553,7 @@ ], [ "macro_expansion", -"daf3d1983cebe5940667a1b2" +"33f5c308ce06bde2928df151" ], [ "macro_expansion", @@ -7589,7 +7589,7 @@ ], [ "macro_expansion", -"1f6de1de7c080616a4830b38" +"4662f579661dd54d015bb345" ], [ "macro_expansion", @@ -7749,7 +7749,7 @@ ], [ "macro_expansion", -"79f966465322ad3fcf1b36e0" +"3a9b3bf6a87fc41ff0db42cb" ], [ "macro_expansion", @@ -7849,7 +7849,7 @@ ], [ "macro_expansion", -"4787957c4eef4cff14d0377c" +"67e1f0b0749acacc059a2c76" ], [ "macro_expansion", @@ -7865,7 +7865,7 @@ ], [ "macro_expansion", -"53f234c92669230f8c720e6f" +"d0ec96a62704cef2870ff8f9" ], [ "macro_expansion", @@ -7885,7 +7885,7 @@ ], [ "macro_expansion", -"2e67cd79331ad972e5208348" +"ed1c94ee796635343fb4f5c5" ], [ "macro_expansion", @@ -7905,7 +7905,7 @@ ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -7925,7 +7925,7 @@ ], [ "macro_expansion", -"1c8bc147c4990d95b0a6b92f" +"db56663a53aea7e96b33b09a" ], [ "macro_expansion", @@ -7941,7 +7941,7 @@ ], [ "macro_expansion", -"07fb3f2526c844de51ccb111" +"32daf33ba7b807495f6a0e2d" ], [ "macro_expansion", @@ -8093,7 +8093,7 @@ ], [ "macro_expansion", -"98cc4fba902dc3eb859321a2" +"1d1cd7942c4a9a11568c8197" ], [ "macro_expansion", @@ -8109,11 +8109,11 @@ ], [ "macro_expansion", -"ffd6a0fa2d65014b0c860d5e" +"107c71f072cb9739afb7c4c0" ], [ "macro_expansion", -"8ae15f931aa339c6b4603ac7" +"44e7013d06ed4dea1643b8c6" ], [ "macro_expansion", @@ -8149,7 +8149,7 @@ ], [ "macro_expansion", -"8011449f1222ad7a5f71c448" +"ce7eb408d122f5f1316ba6f5" ], [ "macro_expansion", @@ -8165,7 +8165,7 @@ ], [ "macro_expansion", -"2949ea8aa52945d60d48f6c8" +"44fc6b315dea55811832a578" ], [ "macro_expansion", @@ -8197,7 +8197,7 @@ ], [ "macro_expansion", -"7d26346d5b12f684fc69d6aa" +"da6f29094070c73c52b05e90" ], [ "macro_expansion", @@ -8277,7 +8277,7 @@ ], [ "macro_expansion", -"f0c23d46cf2e332ba4ee21bb" +"d742cfb0fc174faaf244e782" ], [ "macro_expansion", @@ -8297,7 +8297,7 @@ ], [ "macro_expansion", -"a70610217967542ced9e7879" +"468892752a20f3cd6e324f68" ], [ "macro_expansion", @@ -8333,7 +8333,7 @@ ], [ "macro_expansion", -"845cdc09ac8154bd9da47f6e" +"988bc5340f1daed7391234e9" ], [ "macro_expansion", @@ -8361,7 +8361,7 @@ ], [ "macro_expansion", -"530ddd446af6879034f7e465" +"3fe9f38ab55692b3a0a961bc" ], [ "macro_expansion", @@ -8389,7 +8389,7 @@ ], [ "macro_expansion", -"3ffbc772ee3c92adfe7323a6" +"ca36c4427550ca7e376697b4" ], [ "macro_expansion", @@ -8413,7 +8413,7 @@ ], [ "macro_expansion", -"f3fdabdefe61300e4ea086a1" +"af04dd4f3d7e51b444ca50c1" ], [ "macro_expansion", @@ -8441,7 +8441,7 @@ ], [ "macro_expansion", -"5984bafd0b9665162a9cbe88" +"5bf2863fd2a66208d8538134" ], [ "macro_expansion", @@ -8477,7 +8477,7 @@ ], [ "macro_expansion", -"315b6f1c1a90acd3005747de" +"500a2a152cbf09ab0f441142" ], [ "macro_expansion", @@ -8513,7 +8513,7 @@ ], [ "macro_expansion", -"b8cd61e09c3d41c8f274aff3" +"b33f5a542dbca974751b3876" ], [ "macro_expansion", @@ -8533,7 +8533,7 @@ ], [ "macro_expansion", -"a611d0bcce6abd48db137abb" +"e95ec76e936ad2d64dab1bb4" ], [ "macro_expansion", @@ -8565,7 +8565,7 @@ ], [ "macro_expansion", -"6f7009a267639c1d222d1e38" +"b02e4aedd2ab9281e6f5b2f8" ], [ "macro_expansion", @@ -8665,11 +8665,11 @@ ], [ "macro_expansion", -"2435576858b7d1799574e06a" +"1a0c3676e0b04fd440a5dd4d" ], [ "macro_expansion", -"4124989d154c040254f2b059" +"dedf36970e8bffe090cb2202" ], [ "macro_expansion", @@ -8677,7 +8677,7 @@ ], [ "macro_expansion", -"4a057747c3183d1281e00320" +"f13b8de2d8306b22dcc2c082" ], [ "macro_expansion", @@ -8693,11 +8693,11 @@ ], [ "macro_expansion", -"383b040c50ed5f674863f953" +"548da1f4f425cb2612d2f541" ], [ "macro_expansion", -"190422e2dcd6f4fe2a6c345b" +"be933d406e2aced6df9fc747" ], [ "macro_expansion", @@ -8709,7 +8709,7 @@ ], [ "macro_expansion", -"66019913301212be2d1e4121" +"b6c56aa4ae97b60d83a6a13f" ], [ "macro_expansion", @@ -8741,19 +8741,19 @@ ], [ "macro_expansion", -"3025298564ade8c3e4b6ce77" +"237cb4cd4e84fdfc8ff2e868" ], [ "macro_expansion", -"2562f67816fd5657d6483702" +"b1047e887445d52010961edf" ], [ "macro_expansion", -"81f2d6cbed93eeda2395697c" +"ce87ce10f32ff07a9d88a6cb" ], [ "macro_expansion", -"b4399d0f56dc39a56b4132b5" +"2460e2c47d58b517899ff1cd" ], [ "macro_expansion", @@ -8761,7 +8761,7 @@ ], [ "macro_expansion", -"7ededa26e6d57fac1787b7b7" +"34c254225d33b6d6022cd526" ], [ "macro_expansion", @@ -8825,7 +8825,7 @@ ], [ "macro_expansion", -"d177d6f52304ba6f3fea1d94" +"2c7cc6124c6144ec8f93e43e" ], [ "macro_expansion", @@ -8853,7 +8853,7 @@ ], [ "macro_expansion", -"cecc02b13e33a3a296d2995d" +"bec48008f9471aff4bf12356" ], [ "macro_expansion", @@ -8881,11 +8881,11 @@ ], [ "macro_expansion", -"8c02dafcd36e523c6a46aabb" +"389a4a632472125718547492" ], [ "macro_expansion", -"0ca16da0fb0c2abc8427f886" +"996998eba4872d107d2b5b93" ], [ "macro_expansion", @@ -8925,7 +8925,7 @@ ], [ "macro_expansion", -"14703918ddb97f26dd1df75c" +"244c0695a4ade084211d818a" ], [ "macro_expansion", @@ -8953,11 +8953,11 @@ ], [ "macro_expansion", -"c14280d615907b886813edd7" +"d61fd269cf1493fe5f760adc" ], [ "macro_expansion", -"1da96a08f0b70177f9f153da" +"d42bb0dcf0471c4c9cbf5bfa" ], [ "macro_expansion", @@ -8969,7 +8969,7 @@ ], [ "macro_expansion", -"1ae4baa5d1cca350f4cfc336" +"1baf6d1dc767ea51486986c9" ], [ "macro_expansion", @@ -9045,7 +9045,7 @@ ], [ "macro_expansion", -"a87d60aab5eed2ac32b77274" +"ea4b7826a777adeaadb70d06" ], [ "macro_expansion", @@ -9065,7 +9065,7 @@ ], [ "macro_expansion", -"f1ad42ccc4f12cce970a303d" +"18ea958ca7af8af58643c85c" ], [ "macro_expansion", @@ -9073,7 +9073,7 @@ ], [ "macro_expansion", -"17d380cb93d22b94b76607b7" +"6daa8df354f9a4b8e04a8cb1" ], [ "macro_expansion", @@ -9169,7 +9169,7 @@ ], [ "macro_expansion", -"a7b17da60947144cdadb2a60" +"0cf4ed86b9d7aa3b63f357b0" ], [ "macro_expansion", @@ -9181,11 +9181,11 @@ ], [ "macro_expansion", -"da428a95f0d01dcdce1ac1f9" +"f1ad3101056ae46ffbaa1f31" ], [ "macro_expansion", -"9a9138f6883ec1580ff0a0f1" +"ea61212fa00218113caca76d" ], [ "macro_expansion", @@ -9197,7 +9197,7 @@ ], [ "macro_expansion", -"afa945afb66b78ee0cda0f71" +"41eba032d2ac0588a9683698" ], [ "macro_expansion", @@ -9341,7 +9341,7 @@ ], [ "macro_expansion", -"43d5b652818979382770a9bd" +"1b0075899d87618b666feaf2" ], [ "macro_expansion", @@ -9365,7 +9365,7 @@ ], [ "macro_expansion", -"603ae532f184abcb959d5c38" +"350b215df6e392d33b53bee1" ], [ "macro_expansion", @@ -9401,7 +9401,7 @@ ], [ "macro_expansion", -"32ecbbb5dc5bbe6c96049fea" +"3f9087a3f5bbdf23efb8d83b" ], [ "macro_expansion", @@ -9445,7 +9445,7 @@ ], [ "macro_expansion", -"1a4ea9d43b91c6645cb888ca" +"e0b916b227ff71d935dbcd5f" ], [ "macro_expansion", @@ -9477,7 +9477,7 @@ ], [ "macro_expansion", -"cb396ea48c667b278d2ed568" +"c85dfe428242e2164ab62c38" ], [ "macro_expansion", @@ -9517,7 +9517,7 @@ ], [ "macro_expansion", -"e82cd61f906361b988a85378" +"fb8a0f5f2deda989da476eae" ], [ "macro_expansion", @@ -9573,7 +9573,7 @@ ], [ "macro_expansion", -"e82a279dc0553426cd3cec6c" +"467772b4894b01673d10159c" ], [ "macro_expansion", @@ -9597,7 +9597,7 @@ ], [ "macro_expansion", -"d45c608965100b7e6f319517" +"0f1034bffb0f396a8a052b80" ], [ "macro_expansion", @@ -9725,7 +9725,7 @@ ], [ "macro_expansion", -"735e85a7fb1c732b972cef86" +"4f2b2909cdcdbff542a8a119" ], [ "macro_expansion", @@ -9757,7 +9757,7 @@ ], [ "macro_expansion", -"4d4a5643bba3fffa1f8cdf50" +"1b5912b56aade8a1ffcf6516" ], [ "macro_expansion", @@ -9785,7 +9785,7 @@ ], [ "macro_expansion", -"0e3a012250cead63111b0ce3" +"28457987fc705874c6f0ec56" ], [ "macro_expansion", @@ -9893,7 +9893,7 @@ ], [ "macro_expansion", -"797e27b2fa9c87192759256f" +"3821960fc232ec6492638392" ], [ "macro_expansion", @@ -9941,7 +9941,7 @@ ], [ "macro_expansion", -"08823df548a9fe1b9191ad48" +"a9822e3dc2d7d05d38c0e837" ], [ "macro_expansion", @@ -9997,7 +9997,7 @@ ], [ "macro_expansion", -"61f0f521a7b07ec1c6961a98" +"c91d56b76923309c1056ec0c" ], [ "macro_expansion", @@ -10009,11 +10009,11 @@ ], [ "macro_expansion", -"19600f031149b2fb0ffe8fa7" +"d5524bf9a7b8c097ac76253b" ], [ "macro_expansion", -"89bbd04e4fa3da1ebd691309" +"67efe9c915d2e593ac030a74" ], [ "macro_expansion", @@ -10145,7 +10145,7 @@ ], [ "macro_expansion", -"5f6fc192aa8affb6961e16c4" +"da0d91201cbbd7b72ff30b81" ], [ "macro_expansion", @@ -10189,7 +10189,7 @@ ], [ "macro_expansion", -"42c73e56504e281951acb3cf" +"4be0d7c0eca7b9b1b272296f" ], [ "macro_expansion", @@ -10269,7 +10269,7 @@ ], [ "macro_expansion", -"92a4fab4888f402ca38904a2" +"25427c5970b69fdca8c7bee3" ], [ "macro_expansion", @@ -10309,7 +10309,7 @@ ], [ "macro_expansion", -"18045af0affec44783ee02fb" +"cb37caf3eb0f636bbd0ee6db" ], [ "macro_expansion", @@ -10329,7 +10329,7 @@ ], [ "macro_expansion", -"f68cb2ad9d35e11f2163ebd7" +"598f5b9f93661a729a05f1dd" ], [ "macro_expansion", @@ -10337,7 +10337,7 @@ ], [ "macro_expansion", -"9aa491448b43f4ce6c11f21e" +"dee585f1cacd4c8de212849e" ], [ "macro_expansion", @@ -10349,7 +10349,7 @@ ], [ "macro_expansion", -"79dac5598e9d65ad13135abf" +"3707829e9a9d505b7f59e73a" ], [ "macro_expansion", @@ -10357,7 +10357,7 @@ ], [ "macro_expansion", -"036359f7919522149aef8f4f" +"b8377f3f3081da385ccffa46" ], [ "macro_expansion", @@ -10629,11 +10629,11 @@ ], [ "macro_expansion", -"97a35633a8086c5976440e79" +"cd4164f2a444dfc530fb6463" ], [ "macro_expansion", -"7192c008fbf8553cd48e49f9" +"6e11d3b4bca36197795ba2b9" ], [ "macro_expansion", @@ -10657,7 +10657,7 @@ ], [ "macro_expansion", -"484d56bae1424c453651cf14" +"303e05253da2656adc4c2b71" ], [ "macro_expansion", @@ -10725,7 +10725,7 @@ ], [ "macro_expansion", -"da912b9d4f568447cdc08116" +"24e56ba1a055dd15e873ab1e" ], [ "macro_expansion", @@ -10797,7 +10797,7 @@ ], [ "macro_expansion", -"dddf718ac51c76469ac8917b" +"ad9fedbf236b675656580a3a" ], [ "macro_expansion", @@ -10841,7 +10841,7 @@ ], [ "macro_expansion", -"61182e0e1e0648487407c7b3" +"bf7e88838ef4b35f53a685db" ], [ "macro_expansion", @@ -10869,7 +10869,7 @@ ], [ "macro_expansion", -"6ce8023c88d488f90921a168" +"850b76d52e19dd1e80c372a5" ], [ "macro_expansion", @@ -11017,7 +11017,7 @@ ], [ "macro_expansion", -"c1a47ae79400abe44c230355" +"4458dfcf434e54d03d724030" ], [ "macro_expansion", @@ -11045,7 +11045,7 @@ ], [ "macro_expansion", -"30664fa2f1a76759e81a4096" +"9236091e584b23095da7470f" ], [ "macro_expansion", @@ -11241,7 +11241,7 @@ ], [ "macro_expansion", -"0a71563f1bf00afcd0466607" +"5bc69962295a8029cc6ab551" ], [ "macro_expansion", @@ -11317,7 +11317,7 @@ ], [ "macro_expansion", -"d1f9ffad1bfd252ab90c06cc" +"16006b29f6000e58894d096d" ], [ "macro_expansion", @@ -11349,7 +11349,7 @@ ], [ "macro_expansion", -"5a44715646d4fb92ecd95da6" +"56969a87519250b91077f3cc" ], [ "macro_expansion", @@ -11409,7 +11409,7 @@ ], [ "macro_expansion", -"65e9ddc3f0676c38fb4c5afb" +"634a1cd59ee5b19257dc5738" ], [ "macro_expansion", @@ -11417,7 +11417,7 @@ ], [ "macro_expansion", -"c705ec360fadcc4f78f8249c" +"1c9dd1a4a1410b6a0bc58834" ], [ "macro_expansion", @@ -11449,11 +11449,11 @@ ], [ "macro_expansion", -"5eb32b2addb16175a9674315" +"7a948b93666cad9a63e20bf5" ], [ "macro_expansion", -"61ab9f76bd2a3e040e08ab58" +"1a37391ea5a0ff4cef7481c9" ], [ "macro_expansion", @@ -11493,7 +11493,7 @@ ], [ "macro_expansion", -"1b204811806014fbf0834312" +"e4676d0f1e2cc6063be4cfe1" ], [ "macro_expansion", @@ -11501,7 +11501,7 @@ ], [ "macro_expansion", -"0035555c6180a6a1345c0ae4" +"5a08194052dc54162bd044be" ], [ "macro_expansion", @@ -11509,7 +11509,7 @@ ], [ "macro_expansion", -"ac2f9545406d9cf88e687409" +"425c39e912fc6552dfaa2a43" ], [ "macro_expansion", @@ -11529,7 +11529,7 @@ ], [ "macro_expansion", -"6c23beb1304450895091b5b4" +"72decc3c090b9f40e7a9d936" ], [ "macro_expansion", @@ -11545,7 +11545,7 @@ ], [ "macro_expansion", -"9c591cf40b4e745d27107df5" +"0a115a225f4b39de8b5ca42f" ], [ "macro_expansion", @@ -11781,7 +11781,7 @@ ], [ "macro_expansion", -"ab69683bd4cd54d6bcc2ae02" +"6135a252f9ea15e0657ae82c" ], [ "macro_expansion", @@ -11841,7 +11841,7 @@ ], [ "macro_expansion", -"7b2bfcfde3fdc7a18e68359a" +"014c8a087216c34ed4dbfb64" ], [ "macro_expansion", @@ -11873,7 +11873,7 @@ ], [ "macro_expansion", -"f6ad9bec6c2745d27265d4ce" +"893cd7ac13c47a9e50947740" ], [ "macro_expansion", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index afaa800b44..e95d14980c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -116,6 +116,15 @@ class NativeFunctionAttachmentTestCase: expected_usages: tuple[str, ...] +@dataclass(frozen=True) +class PythonTableFunctionCompileTestCase: + """Python table functions and the columns and frame each compiles to.""" + + description: str + files: dict[str, str] + expected_summaries: dict[str, tuple[object, ...]] + + @dataclass(frozen=True) class NativeFunctionErrorTestCase: """Function files the native loop rejects while attaching them.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json index dc66150d4c..781f00856b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json @@ -16,7 +16,7 @@ "macro_source_sql": "'PIVOT __ref(\"order_amounts\") ON category USING MAX(amount)'", "macro_usages": "()", "query_sql": "'PIVOT __ref(\"order_amounts\") ON category USING MAX(amount)'", - "references": "(CompileSqlReference(ref_kind=, ref_name='order_amounts', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='order_amounts', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='category_amounts', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='customer_id', type='INTEGER', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/category_amounts.sql'), line=2, column=12, end_line=2, end_column=23), migrate_from=None),), dynamic_columns=(SchemaDynamicColumnFamily(name='category_amounts', pivot_column='category', value_column='amount', aggregate='MAX', type='INTEGER', name_pattern='^[a-z]+$'),), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='PIVOT __ref(\"order_amounts\") ON category USING MAX(amount)', expanded_sql='PIVOT __ref(\"order_amounts\") ON category USING MAX(amount)', passes=((), ()))", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json index 5a954c2a0f..d847394db3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json @@ -86,7 +86,7 @@ "macro_source_sql": "'SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()'", "macro_usages": "()", "query_sql": "'SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='orders', model_schema=None, description='Orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', passes=((), ()))", @@ -103,7 +103,7 @@ "macro_source_sql": "'SELECT @py.cents(\\'3\\') AS cents, order_id FROM __ref(\"stg_orders\")'", "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='priced'), declaration=DeclarationIdentity(kind=, name='cents', owner=None), kind=, through=None, enum_member=None),)", "query_sql": "'SELECT 3 * 100 AS cents, order_id FROM __ref(\"stg_orders\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='priced', model_schema=None, description='Priced orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT @py.cents(\\'3\\') AS cents, order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT 3 * 100 AS cents, order_id FROM __ref(\"stg_orders\")', passes=((), (ExpansionSpan(source_start=7, source_end=21, output_start=7, output_end=14),)))", @@ -154,7 +154,7 @@ "macro_source_sql": "'SELECT method FROM __seed(\"payment_methods\")'", "macro_usages": "()", "query_sql": "'SELECT method FROM __seed(\"payment_methods\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='payment_methods', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='payment_methods', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "None", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT method FROM __seed(\"payment_methods\")', expanded_sql='SELECT method FROM __seed(\"payment_methods\")', passes=((), ()))", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json index d8e7ff4d18..6a2315741c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json @@ -15,7 +15,7 @@ "macro_source_sql": "'SELECT order_id FROM __ref(\"stg_orders\")'", "macro_usages": "()", "query_sql": "'SELECT order_id FROM __ref(\"stg_orders\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "None", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")', passes=((), ()))", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json index 295f84a9c3..be6dcf27d9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json @@ -12,7 +12,7 @@ "macro_source_sql": "'SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id'", "macro_usages": "()", "query_sql": "'SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id'", - "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='customer_status_snapshot', model_schema=None, description='Merge-based customer snapshot with timestamp cursor and explicit backfill policies.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/intermediate/customer_status_snapshot.sql'), line=16, column=5, end_line=16, end_column=16), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', expanded_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', passes=((), ()))", @@ -29,7 +29,7 @@ "macro_source_sql": "'SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")'", "macro_usages": "()", "query_sql": "'SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='order_status_index', model_schema=None, description='Integer-cursor incremental projection over fact orders.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/intermediate/order_status_index.sql'), line=11, column=5, end_line=11, end_column=13), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', expanded_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', passes=((), ()))", @@ -46,7 +46,7 @@ "macro_source_sql": "'SELECT\\n @py.timestamp_trunc(\\'day\\', \\'activity_hour\\') AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY @py.timestamp_trunc(\\'day\\', \\'activity_hour\\')'", "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='daily_activity_rollup'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", "query_sql": "'SELECT\\n DATE_TRUNC(\\'day\\', activity_hour) AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY DATE_TRUNC(\\'day\\', activity_hour)'", - "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='daily_activity_rollup', model_schema=None, description='Downstream daily microbatch rollup with a wider batch size than its hourly upstream.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_day', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/marts/daily_activity_rollup.sql'), line=18, column=5, end_line=18, end_column=17), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'orders_placed >= 0'}, name='daily_orders_placed_is_non_negative', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @py.timestamp_trunc(\\'day\\', \\'activity_hour\\') AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY @py.timestamp_trunc(\\'day\\', \\'activity_hour\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'day\\', activity_hour) AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY DATE_TRUNC(\\'day\\', activity_hour)', passes=((), (ExpansionSpan(source_start=9, source_end=52, output_start=9, output_end=41), ExpansionSpan(source_start=235, source_end=278, output_start=224, output_end=256))))", @@ -63,7 +63,7 @@ "macro_source_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)'", "macro_usages": "()", "query_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='daily_order_partitioned', model_schema=None, description='Partition-tracked daily order summary using custom materialization.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_date', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/marts/daily_order_partitioned.sql'), line=16, column=5, end_line=16, end_column=15), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'waffles_ordered > 0'}, name='waffles_ordered_is_positive', description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((), ()))", @@ -80,7 +80,7 @@ "macro_source_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)'", "macro_usages": "()", "query_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None, call_partitioned=False))", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='daily_revenue', model_schema=None, description='Daily revenue aggregation.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='revenue_date', type=None, nullable=False, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/daily_revenue.sql'), line=12, column=5, end_line=12, end_column=17), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'total_revenue_cents >= 0'}, name='revenue_is_non_negative', description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((), ()))", @@ -97,7 +97,7 @@ "macro_source_sql": "'SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at'", "macro_usages": "()", "query_sql": "'SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_customers', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_customers', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None, call_partitioned=False))", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='dim_customers', model_schema=None, description='Customer dimension with lifetime metrics.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/marts/dim_customers.sql'), line=6, column=5, end_line=6, end_column=16), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', expanded_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', passes=((), ()))", @@ -114,7 +114,7 @@ "macro_source_sql": "'SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n @py.line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='fact_orders'), declaration=DeclarationIdentity(kind=, name='line_total_cents', owner=None), kind=, through=None, enum_member=None),)", "query_sql": "'SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n w.price_cents * o.quantity AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", - "references": "(CompileSqlReference(ref_kind=, ref_name='is_completed_order', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='is_completed_order_py', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "references": "(CompileSqlReference(ref_kind=, ref_name='is_completed_order', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='is_completed_order_py', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None, call_partitioned=False))", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='fact_orders', model_schema=None, description='Order fact table with waffle and payment details.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/marts/fact_orders.sql'), line=6, column=5, end_line=6, end_column=13), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n @py.line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n w.price_cents * o.quantity AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), (ExpansionSpan(source_start=124, source_end=175, output_start=124, output_end=150),)))", @@ -131,7 +131,7 @@ "macro_source_sql": "'SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON @py.timestamp_trunc(\\'day\\', \\'h.activity_hour\\') = d.activity_day'", "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hourly_activity_with_daily_context'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", "query_sql": "'SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON DATE_TRUNC(\\'day\\', h.activity_hour) = d.activity_day'", - "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='daily_activity_rollup', ref_package=None, call_argument_count=None))", + "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='daily_activity_rollup', ref_package=None, call_argument_count=None, call_partitioned=False))", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='hourly_activity_with_daily_context', model_schema=None, description='Hourly downstream microbatch model that depends on a coarser-grain daily upstream.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_hour', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/marts/hourly_activity_with_daily_context.sql'), line=17, column=5, end_line=17, end_column=18), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'day_orders_placed >= orders_placed'}, name='day_orders_cover_hourly_orders', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON @py.timestamp_trunc(\\'day\\', \\'h.activity_hour\\') = d.activity_day', expanded_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON DATE_TRUNC(\\'day\\', h.activity_hour) = d.activity_day', passes=((), (ExpansionSpan(source_start=211, source_end=256, output_start=211, output_end=245),)))", @@ -148,7 +148,7 @@ "macro_source_sql": "'SELECT\\n @py.timestamp_trunc(\\'hour\\', \\'o.ordered_at\\') AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY @py.timestamp_trunc(\\'hour\\', \\'o.ordered_at\\')'", "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hourly_order_activity'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", "query_sql": "'SELECT\\n DATE_TRUNC(\\'hour\\', o.ordered_at) AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY DATE_TRUNC(\\'hour\\', o.ordered_at)'", - "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='hourly_order_activity', model_schema=None, description='Hourly order activity aggregated via microbatch incremental.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_hour', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/marts/hourly_order_activity.sql'), line=17, column=5, end_line=17, end_column=18), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'orders_placed >= 0'}, name='orders_placed_is_non_negative', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @py.timestamp_trunc(\\'hour\\', \\'o.ordered_at\\') AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY @py.timestamp_trunc(\\'hour\\', \\'o.ordered_at\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'hour\\', o.ordered_at) AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY DATE_TRUNC(\\'hour\\', o.ordered_at)', passes=((), (ExpansionSpan(source_start=9, source_end=52, output_start=9, output_end=41), ExpansionSpan(source_start=218, source_end=261, output_start=207, output_end=239))))", @@ -165,7 +165,7 @@ "macro_source_sql": "'SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", "macro_usages": "()", "query_sql": "'SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", - "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None, call_partitioned=False), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None, call_partitioned=False))", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='scenario_order_prices', model_schema=None, description='Small model used by scenario CLI e2e retain coverage.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), ()))", @@ -182,7 +182,7 @@ "macro_source_sql": "'SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")'", "macro_usages": "()", "query_sql": "'SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='raw_customers', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_customers', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='stg_customers', model_schema=None, description='Cleaned customer records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/staging/stg_customers.sql'), line=6, column=5, end_line=6, end_column=16), migrate_from=None), SchemaColumn(name='email', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/staging/stg_customers.sql'), line=7, column=5, end_line=7, end_column=10), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', expanded_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', passes=((), ()))", @@ -199,7 +199,7 @@ "macro_source_sql": "'SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")'", "macro_usages": "()", "query_sql": "'SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='raw_orders', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_orders', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='stg_orders', model_schema=None, description='Cleaned order records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=6, column=5, end_line=6, end_column=13), migrate_from=None), SchemaColumn(name='customer_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=7, column=5, end_line=7, end_column=16), migrate_from=None), SchemaColumn(name='waffle_type_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='relationships', arguments={'to': '__seed(\"waffle_types\")', 'field': 'waffle_type_id'}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=8, column=5, end_line=8, end_column=19), migrate_from=None), SchemaColumn(name='status', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='accepted_values', arguments={'values': ['placed', 'preparing', 'ready', 'completed', 'cancelled']}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=11, column=5, end_line=11, end_column=11), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', expanded_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', passes=((), ()))", @@ -216,7 +216,7 @@ "macro_source_sql": "'SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")'", "macro_usages": "()", "query_sql": "'SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")'", - "references": "(CompileSqlReference(ref_kind=, ref_name='raw_payments', ref_package=None, call_argument_count=None),)", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_payments', ref_package=None, call_argument_count=None, call_partitioned=False),)", "rejected_sql_analysis_opt_out": "None", "schema_entry": "SchemaModelEntry(name='stg_payments', model_schema=None, description='Cleaned payment records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='payment_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/staging/stg_payments.sql'), line=6, column=5, end_line=6, end_column=15), migrate_from=None), SchemaColumn(name='order_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None), SchemaAuditInstance(definition_name='relationships', arguments={'to': '__source(\"raw_orders\")', 'field': 'id'}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None, meta=None)), location=SourceLocation(path=PosixPath('models/staging/stg_payments.sql'), line=7, column=5, end_line=7, end_column=13), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', expanded_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', passes=((), ()))", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 77e2fe33ad..5454370e9e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -39,6 +39,7 @@ ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import ( CompiledAuditView, CompiledFunctionView, @@ -572,6 +573,32 @@ def column_metadata(column: SourceColumnEntry) -> tuple[str | None, tuple[dict[s } +TABLE_FUNCTION_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "snowflake"\n', + "models/readings.sql": ( + "MODEL (description 'Readings.');\nSELECT 1 AS device_id, 1.0 AS ts, 2.0 AS reading\n" + ), + "functions/python/smooth.py": ( + "import polars as pl\nfrom sqlbuild.functions import table_fn\n\n" + "@table_fn(arguments={'device_id': 'INTEGER', 'reading': 'FLOAT'}," + " returns={'device_id': 'INTEGER', 'smoothed': 'FLOAT'}, runtime_version='3.12')\n" + "def main(readings: pl.DataFrame) -> pl.DataFrame:\n" + ' """Smooth."""\n return readings\n' + ), + "functions/python/rank_readings.py": ( + "import pandas as pd\nfrom sqlbuild.functions import table_fn\n\n" + "@table_fn(arguments={'reading': 'FLOAT'}, returns={'position': 'INTEGER'}," + " runtime_version='3.12')\n" + "def main(readings: pd.DataFrame) -> pd.DataFrame:\n" + ' """Rank."""\n return readings\n' + ), + "models/smoothed.sql": ( + "MODEL (description 'Smoothed.');\nSELECT s.* FROM __ref(\"readings\") r,\n" + ' __table_fn("smooth")(r.device_id, r.reading) OVER (PARTITION BY r.device_id) s\n' + ), +} + + def compile_function_project( *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch ) -> tuple[CompiledProjectView, CompileInputsView]: @@ -587,6 +614,30 @@ def compile_function_project( return project, inputs +def compile_snowflake_project(*, project_dir: Path, files: dict[str, str]) -> CompiledProjectView: + """Compile a Snowflake project of `files`, the adapter Python table functions deploy on.""" + + write_files(project_dir=project_dir, files=files) + return build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=SnowflakeAdapter(), + resolved_connection=_SOURCE_CONNECTION, + ) + + +def table_function_summary(function: CompiledFunctionView) -> tuple[object, ...]: + """A compiled Python table function's kind, columns and partition frame.""" + + return ( + str(function.key.resource_type), + function.language.value, + function.returns, + tuple((column.name, column.type) for column in function.return_columns), + function.entry_point, + function.dataframe, + ) + + def function_summary(function: CompiledFunctionView) -> tuple[object, ...]: """A compiled function's planner-facing facts.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py index bd2c76ed63..5a6503a97d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py @@ -8,21 +8,41 @@ import pytest from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledFunctionView +from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeFunctionAttachmentTestCase, NativeFunctionErrorTestCase, + PythonTableFunctionCompileTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, + TABLE_FUNCTION_PROJECT_FILES, compile_function_project, + compile_snowflake_project, function_summary, + table_function_summary, ) if TYPE_CHECKING: from sqlbuild._native import FunctionInput +_SMOOTH_SUMMARY: tuple[object, ...] = ( + "table_fn", + "python", + "TABLE", + (("device_id", "INTEGER"), ("smoothed", "FLOAT")), + "main", +) +_RANK_SUMMARY: tuple[object, ...] = ( + "table_fn", + "python", + "TABLE", + (("position", "INTEGER"),), + "main", +) + + @pytest.mark.parametrize( "test_case", [ @@ -154,3 +174,116 @@ def test_given_invalid_function_when_compiling_then_native_attachment_rejects_it ) assert test_case.expected_message in str(raised.value) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTableFunctionCompileTestCase( + description="polars by default and pandas from a pd.DataFrame annotation", + files=TABLE_FUNCTION_PROJECT_FILES, + expected_summaries={ + "smooth": (*_SMOOTH_SUMMARY, "polars"), + "rank_readings": (*_RANK_SUMMARY, "pandas"), + }, + ), + PythonTableFunctionCompileTestCase( + description="a pandas project default applies to an unannotated function", + files={ + **TABLE_FUNCTION_PROJECT_FILES, + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "snowflake"\n' + '[defaults]\nfunction_dataframe = "pandas"\n' + ), + "functions/python/smooth.py": TABLE_FUNCTION_PROJECT_FILES[ + "functions/python/smooth.py" + ].replace("readings: pl.DataFrame", "readings"), + }, + expected_summaries={ + "smooth": (*_SMOOTH_SUMMARY, "pandas"), + "rank_readings": (*_RANK_SUMMARY, "pandas"), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_table_functions_when_compiling_then_each_keeps_its_columns_and_frame( + test_case: PythonTableFunctionCompileTestCase, tmp_path: Path +) -> None: + project: CompiledProjectView = compile_snowflake_project( + project_dir=tmp_path, files=test_case.files + ) + + assert { + function.name: table_function_summary(function) for function in project.functions + } == test_case.expected_summaries + + +@pytest.mark.parametrize( + "test_case", + [ + NativeFunctionErrorTestCase( + description="a Python table function called without a partition", + files={ + "models/smoothed.sql": ( + "MODEL (description 'Smoothed.');\nSELECT s.* FROM __ref(\"readings\") r,\n" + ' __table_fn("smooth")(r.device_id, r.reading) OVER () s\n' + ), + }, + expected_message=( + "Model file models/smoothed.sql calls Python table function 'smooth' without" + " OVER (PARTITION BY ...)" + ), + ), + NativeFunctionErrorTestCase( + description="a SQL test calling a Python table function without a partition", + files={ + "tests/unit/test_smooth.sql": ( + 'TEST (mode table_fn, name "smooth_rows");\n' + "WITH\nreadings AS (SELECT 1 AS device_id, 2.0::FLOAT AS reading),\n" + "__table_fn_actual__ AS (\n SELECT s.device_id, s.smoothed FROM readings r,\n" + ' __table_fn("smooth")(r.device_id, r.reading) s\n),\n' + "__table_fn_expected__ AS (\n" + " SELECT 1 AS device_id, 2.0::FLOAT AS smoothed\n)\n" + ), + }, + expected_message=( + "SQL test file tests/unit/test_smooth.sql calls Python table function 'smooth'" + " without OVER (PARTITION BY ...)" + ), + ), + NativeFunctionErrorTestCase( + description="a polars table function with a fractional NUMBER argument", + files={ + "functions/python/smooth.py": TABLE_FUNCTION_PROJECT_FILES[ + "functions/python/smooth.py" + ].replace("'reading': 'FLOAT'}", "'reading': 'NUMBER(10, 2)'}"), + }, + expected_message=( + "Python function functions/python/smooth.py argument 'reading' is NUMBER(10, 2)," + " which Snowflake passes to a polars frame as Python Decimal objects" + ), + ), + NativeFunctionErrorTestCase( + description="a table function returning a scalar type", + files={ + "functions/python/rank_readings.py": TABLE_FUNCTION_PROJECT_FILES[ + "functions/python/rank_readings.py" + ].replace("returns={'position': 'INTEGER'}", "returns='INTEGER'"), + }, + expected_message=( + "functions/python/rank_readings.py returns must map each column name to its type" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_python_table_function_when_compiling_then_compile_rejects_it( + test_case: NativeFunctionErrorTestCase, tmp_path: Path +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = compile_snowflake_project( + project_dir=tmp_path, files={**TABLE_FUNCTION_PROJECT_FILES, **test_case.files} + ) + + assert test_case.expected_message in str(raised.value) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 6065bcc473..8c9041d17c 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -129,6 +129,28 @@ class SnowflakeRenderTableFunctionTestCase: expected_sql: str +@dataclass(frozen=True) +class SnowflakeTableFunctionRunTestCase: + description: str + returned_columns: tuple[str, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SnowflakeTableFunctionColumnsErrorTestCase: + description: str + returned_columns: tuple[str, ...] + expected_error: str + + +@dataclass(frozen=True) +class SnowflakePythonTableFunctionTestCase: + description: str + dataframe: str + packages: tuple[str, ...] + expected_sql: str + + @dataclass(frozen=True) class SnowflakeQueryColumnTypesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index f6f14eecfc..48438a7f1f 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -11,7 +11,10 @@ from types import MappingProxyType, ModuleType, SimpleNamespace from typing import Any -from sqlbuild._native import NativeProgressProjector +import pandas +import pytest + +from sqlbuild._native import FunctionArgument, FunctionReturnColumn, NativeProgressProjector from sqlbuild.adapter.contract.models import ( ColumnInfo, RelationInfo, @@ -24,6 +27,7 @@ ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.snowflake.classes.snowflake_connection import _SnowflakeConnection +from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.observability import ( EventDispatcher, LifecycleEvent, @@ -709,3 +713,47 @@ def fail_statement() -> None: """A statement the warehouse rejects.""" raise RuntimeError("statement failed") + + +def run_python_table_function( + *, statement: str, returned_columns: tuple[str, ...], monkeypatch: pytest.MonkeyPatch +) -> tuple[tuple[object, ...], ...]: + """Run a rendered pandas table function's handler on one partition, as Snowflake would.""" + + body: str = statement.split("AS $$\n", 1)[1].rsplit("\n$$", 1)[0] + snowflake_module: ModuleType = ModuleType("_snowflake") + vars(snowflake_module)["vectorized"] = lambda **_: lambda method: method + monkeypatch.setitem(sys.modules, "_snowflake", snowflake_module) + namespace: dict[str, Any] = {"__name__": "udtf"} + exec(body.replace("RETURNED_COLUMNS", repr(list(returned_columns))), namespace) + handler: Any = namespace["_SqbHandler"] + partition: pandas.DataFrame = pandas.DataFrame({0: [2, 1], 1: [10.0, 20.0]}) + result: pandas.DataFrame = handler().end_partition(partition) + return tuple(tuple(row) for row in result.itertuples(index=False)) + + +def pandas_order_totals_statement() -> str: + """The Snowflake DDL of a pandas table function `result` doubling each order amount.""" + + return SnowflakeAdapter().render_create_function( + destination="analytics.order_totals", + arguments=( + FunctionArgument(name="ts", type="INTEGER"), + FunctionArgument(name="amount", type="FLOAT"), + ), + returns="TABLE", + body_sql=( + "def result(readings):\n" + " readings = readings.sort_values('ts')\n" + " readings['total'] = readings['amount'] * 2\n" + " return readings[RETURNED_COLUMNS]" + ), + return_columns=( + FunctionReturnColumn(name="ts", type="INTEGER"), + FunctionReturnColumn(name="total", type="FLOAT"), + ), + language=FunctionLanguage.PYTHON, + runtime_version="3.12", + entry_point="result", + dataframe="pandas", + )[0] diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 0f5530e02f..30238db7f6 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -37,6 +37,7 @@ SnowflakeMoveOrCopyRelationTestCase, SnowflakePlatformDetectionTestCase, SnowflakePruneSqlTestCase, + SnowflakePythonTableFunctionTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, SnowflakeQueryColumnTypesTestCase, @@ -53,6 +54,8 @@ SnowflakeTableFreshnessMetadataErrorTestCase, SnowflakeTableFreshnessMetadataTestCase, SnowflakeTableFreshnessOutcomeTestCase, + SnowflakeTableFunctionColumnsErrorTestCase, + SnowflakeTableFunctionRunTestCase, SnowflakeTableTypeDdlTestCase, ) from tests.unit.src.sqlbuild.adapters.snowflake.helpers import ( @@ -71,6 +74,8 @@ describe_equivalent_numeric_relation, expected_qualified_columns, install_fake_snowflake_connector, + pandas_order_totals_statement, + run_python_table_function, ) @@ -1162,6 +1167,189 @@ def test_given_table_function_when_rendering_then_snowflake_returns_expected_ddl assert statements == (test_case.expected_sql,) +PYTHON_TABLE_FUNCTION_BODY: str = ( + "def main(readings):\n" + " readings = readings.sort_values('ts')\n" + " readings['total'] = readings['amount'] * 2\n" + " return readings[RETURNED_COLUMNS]" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakePythonTableFunctionTestCase( + description="a polars function converts each partition and adds polars and pyarrow", + dataframe="polars", + packages=("numpy", "polars==1.20.0"), + expected_sql=( + "CREATE OR REPLACE FUNCTION analytics.order_totals(ts INTEGER, amount FLOAT)\n" + "RETURNS TABLE (ts INTEGER, total FLOAT)\n" + "LANGUAGE PYTHON\n" + "RUNTIME_VERSION = '3.12'\n" + "HANDLER = '_SqbHandler'\n" + "PACKAGES = ('numpy','polars==1.20.0','pandas','pyarrow')\n" + "AS $$\n" + f"{PYTHON_TABLE_FUNCTION_BODY}\n" + "\n" + "import pandas as _sqb_pandas\n" + "import polars as _sqb_polars\n" + "from _snowflake import vectorized as _sqb_vectorized\n" + "\n" + "\n" + "class _SqbHandler:\n" + " @_sqb_vectorized(input=_sqb_pandas.DataFrame)\n" + " def end_partition(self, _sqb_partition):\n" + " _sqb_partition.columns = ['ts', 'amount']\n" + " _sqb_result = main(_sqb_polars.from_pandas(_sqb_partition))\n" + " _sqb_returned = list(_sqb_result.columns)\n" + " _sqb_expected = ['ts', 'total']\n" + " _sqb_missing = [name for name in _sqb_expected if name not in _sqb_returned]\n" + " _sqb_extra = [name for name in _sqb_returned if name not in _sqb_expected]\n" + " _sqb_repeated = sorted(\n" + " {name for name in _sqb_returned if _sqb_returned.count(name) > 1}\n" + " )\n" + " if _sqb_missing or _sqb_extra or _sqb_repeated:\n" + " raise ValueError(\n" + ' f"table function main returned columns "\n' + ' f"{_sqb_returned}, but its "\n' + ' f"returns declare {_sqb_expected}: missing {_sqb_missing}, "\n' + ' f"extra {_sqb_extra}, repeated {_sqb_repeated}"\n' + " )\n" + " return _sqb_result.select(_sqb_expected).to_pandas()\n" + "$$" + ), + ), + SnowflakePythonTableFunctionTestCase( + description="a pandas function receives the partition as it arrives", + dataframe="pandas", + packages=(), + expected_sql=( + "CREATE OR REPLACE FUNCTION analytics.order_totals(ts INTEGER, amount FLOAT)\n" + "RETURNS TABLE (ts INTEGER, total FLOAT)\n" + "LANGUAGE PYTHON\n" + "RUNTIME_VERSION = '3.12'\n" + "HANDLER = '_SqbHandler'\n" + "PACKAGES = ('pandas')\n" + "AS $$\n" + f"{PYTHON_TABLE_FUNCTION_BODY}\n" + "\n" + "import pandas as _sqb_pandas\n" + "from _snowflake import vectorized as _sqb_vectorized\n" + "\n" + "\n" + "class _SqbHandler:\n" + " @_sqb_vectorized(input=_sqb_pandas.DataFrame)\n" + " def end_partition(self, _sqb_partition):\n" + " _sqb_partition.columns = ['ts', 'amount']\n" + " _sqb_result = main(_sqb_partition)\n" + " _sqb_returned = list(_sqb_result.columns)\n" + " _sqb_expected = ['ts', 'total']\n" + " _sqb_missing = [name for name in _sqb_expected if name not in _sqb_returned]\n" + " _sqb_extra = [name for name in _sqb_returned if name not in _sqb_expected]\n" + " _sqb_repeated = sorted(\n" + " {name for name in _sqb_returned if _sqb_returned.count(name) > 1}\n" + " )\n" + " if _sqb_missing or _sqb_extra or _sqb_repeated:\n" + " raise ValueError(\n" + ' f"table function main returned columns "\n' + ' f"{_sqb_returned}, but its "\n' + ' f"returns declare {_sqb_expected}: missing {_sqb_missing}, "\n' + ' f"extra {_sqb_extra}, repeated {_sqb_repeated}"\n' + " )\n" + " return _sqb_result[_sqb_expected]\n" + "$$" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_table_function_when_rendering_then_snowflake_wraps_it_in_a_partition_handler( + test_case: SnowflakePythonTableFunctionTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_create_function( + destination="analytics.order_totals", + arguments=( + FunctionArgument(name="ts", type="INTEGER"), + FunctionArgument(name="amount", type="FLOAT"), + ), + returns="TABLE", + body_sql=PYTHON_TABLE_FUNCTION_BODY, + return_columns=( + FunctionReturnColumn(name="ts", type="INTEGER"), + FunctionReturnColumn(name="total", type="FLOAT"), + ), + language=FunctionLanguage.PYTHON, + runtime_version="3.12", + entry_point="main", + packages=test_case.packages, + dataframe=test_case.dataframe, + ) + + assert statements == (test_case.expected_sql,) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeTableFunctionRunTestCase( + description=( + "arguments are named and the declared columns return in their order, even for" + " an entry point named like a handler variable" + ), + returned_columns=("total", "ts"), + expected_rows=((1, 40.0), (2, 20.0)), + ) + ], + ids=lambda case: case.description, +) +def test_given_pandas_table_function_when_a_partition_runs_then_declared_columns_return_in_order( + test_case: SnowflakeTableFunctionRunTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rows: tuple[tuple[object, ...], ...] = run_python_table_function( + statement=pandas_order_totals_statement(), + returned_columns=test_case.returned_columns, + monkeypatch=monkeypatch, + ) + + assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeTableFunctionColumnsErrorTestCase( + description="a missing and an extra column are both named", + returned_columns=("ts", "amount"), + expected_error=( + r"table function result returned columns \['ts', 'amount'\], but its returns " + r"declare \['ts', 'total'\]: missing \['total'\], extra \['amount'\], " + r"repeated \[\]" + ), + ), + SnowflakeTableFunctionColumnsErrorTestCase( + description="a column returned twice is named", + returned_columns=("ts", "total", "total"), + expected_error=( + r"table function result returned columns \['ts', 'total', 'total'\], but its " + r"returns declare \['ts', 'total'\]: missing \[\], extra \[\], " + r"repeated \['total'\]" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_pandas_table_function_when_columns_differ_then_missing_and_extra_are_named( + test_case: SnowflakeTableFunctionColumnsErrorTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(ValueError, match=test_case.expected_error): + run_python_table_function( + statement=pandas_order_totals_statement(), + returned_columns=test_case.returned_columns, + monkeypatch=monkeypatch, + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index d1649a493b..3247a1906a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -632,6 +632,7 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: raise ValueError(f"function rendering failed for {destination}") @@ -651,6 +652,7 @@ def render_create_function( runtime_version: str | None = None, entry_point: str | None = None, packages: tuple[str, ...] = (), + dataframe: str | None = None, ) -> tuple[str, ...]: raise KeyboardInterrupt(destination) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 58a18b977e..6dae51dba5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -21,6 +21,7 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, @@ -333,6 +334,55 @@ def test_given_table_function_marker_when_resolving_then_returns_expected_sql( assert result == test_case.expected_sql +@pytest.mark.parametrize( + "test_case", + [ + RefResolutionTestCase( + description="moves a partition clause inside Snowflake's TABLE()", + query_sql=( + 'SELECT s.* FROM r, __table_fn("customer_orders")(r.id) ' + "OVER (PARTITION BY COALESCE(r.region, r.id)) s" + ), + expected_sql=( + "SELECT s.* FROM r, TABLE(analytics.customer_orders(r.id) " + "OVER (PARTITION BY COALESCE(r.region, r.id))) s" + ), + ), + RefResolutionTestCase( + description="keeps a lower-case partition clause across lines", + query_sql='SELECT * FROM r, __table_fn("customer_orders")(r.id)\n over\n(partition by r.id)', + expected_sql="SELECT * FROM r, TABLE(analytics.customer_orders(r.id)\n over\n(partition by r.id))", + ), + RefResolutionTestCase( + description="keeps a comment before the partition clause inside TABLE()", + query_sql='SELECT * FROM r, __table_fn("customer_orders")(r.id) /* c */ OVER (PARTITION BY r.id) s', + expected_sql="SELECT * FROM r, TABLE(analytics.customer_orders(r.id) /* c */ OVER (PARTITION BY r.id)) s", + ), + RefResolutionTestCase( + description="keeps a Snowflake // comment before the partition clause inside TABLE()", + query_sql='SELECT * FROM r, __table_fn("customer_orders")(r.id) // c\n OVER (PARTITION BY r.id) s', + expected_sql="SELECT * FROM r, TABLE(analytics.customer_orders(r.id) // c\n OVER (PARTITION BY r.id)) s", + ), + RefResolutionTestCase( + description="an alias starting with over is not a partition clause", + query_sql='SELECT * FROM __table_fn("customer_orders")(42) overview', + expected_sql="SELECT * FROM TABLE(analytics.customer_orders(42)) overview", + ), + ], + ids=lambda case: case.description, +) +def test_given_partitioned_table_function_when_resolving_for_snowflake_then_partition_is_inside_table( + test_case: RefResolutionTestCase, +) -> None: + result: str = resolve_table_function_references( + query_sql=test_case.query_sql, + function_locations=_FUNCTION_TARGETS, + dialect=Dialect(SnowflakeAdapter()), + ) + + assert result == test_case.expected_sql + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index aa38a2bbee..f00f738166 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -157,6 +157,7 @@ def build_identity_function() -> SimpleNamespace: runtime_version=None, entry_point=None, packages=(), + dataframe=None, ) diff --git a/website/src/content/docs/docs/concepts/functions.mdx b/website/src/content/docs/docs/concepts/functions.mdx index 07df454f94..fce0d7f822 100644 --- a/website/src/content/docs/docs/concepts/functions.mdx +++ b/website/src/content/docs/docs/concepts/functions.mdx @@ -61,7 +61,11 @@ The function name is the file stem (filename without extension). `__udf()` resol ## Table functions -Table functions return multiple rows and columns. They are written in SQL under `functions/sql/` with a `returns table(...)` declaration: +Table functions return multiple rows and columns. Write them in SQL under `functions/sql/` with a `returns table(...)` declaration, or in Python under `functions/python/` with `@table_fn(...)`. + +### SQL table functions + +A SQL table function's body is a query: ```sql -- functions/sql/table_fn__customer_orders.sql @@ -89,6 +93,52 @@ FROM __ref("fact_orders") WHERE customer_id = p_customer_id ``` +### Python table functions + +A Python table function receives one whole partition of its input rows as a data frame and returns a data frame of rows. Use it for per-group numerics that SQL expresses badly, such as smoothing a series or fitting a curve per customer. Place it under `functions/python/`, with exactly one function decorated with `@table_fn(...)`: + +```python +# functions/python/smooth_payments.py +import polars as pl +from sqlbuild.functions import table_fn + +@table_fn( + arguments={"customer_id": "INTEGER", "paid_at": "FLOAT", "amount": "FLOAT"}, + returns={"customer_id": "INTEGER", "paid_at": "FLOAT", "amount_smoothed": "FLOAT"}, + runtime_version="3.12", +) +def main(payments: pl.DataFrame) -> pl.DataFrame: + """Payment amounts smoothed over each customer's last three payments.""" + return payments.sort("paid_at").select( + "customer_id", + "paid_at", + pl.col("amount").rolling_mean(3, min_samples=1).alias("amount_smoothed"), + ) +``` + +Call it with `OVER (PARTITION BY ...)` after the arguments. Each partition is one call of `main`: + +```sql +SELECT s.* +FROM __ref("payments") p, + __table_fn("smooth_payments")(p.customer_id, p.paid_at, p.amount) + OVER (PARTITION BY p.customer_id) s +``` + +- `main` is a plain function: its frame has one column per argument, named after the argument. Test it with pytest like any other Python code. +- The frame is a polars `DataFrame` by default. Annotate the parameter as a pandas `pd.DataFrame` to receive pandas, or set `function_dataframe = "pandas"` under [`[defaults]`](/docs/concepts/project-configuration#defaults) to make pandas the project default. An annotation wins over the default. SQLBuild reads it through the file's top-level `import pandas` or `from pandas import DataFrame`; an annotation in quotes or imported under `if TYPE_CHECKING:` is not read, so the function gets the default. +- SQLBuild generates the warehouse handler and adds the packages the frame needs: `pandas`, and `polars` and `pyarrow` for polars. +- The returned frame must have exactly the `returns` columns. Otherwise the call fails with an error naming the missing, extra and repeated columns. +- Rows arrive in no particular order. Sort inside `main` when order matters. + +| Problem | Fix | +|---------|-----| +| A call without `OVER (PARTITION BY ...)` fails compilation | Add `OVER (PARTITION BY )` after the call | +| A polars function's `NUMBER(p, s)` argument (or `DECIMAL`, `DEC`, `NUMERIC`) with a scale above zero fails compilation | Declare it as `FLOAT`, or as `INTEGER` when it has no fraction | +| `returns` is not a column-to-type mapping, so compilation fails | Map each output column to its type | + +Python table functions deploy on Snowflake only; other adapters reject them when the function is created (see [Adapter support](#adapter-support)). + ### Why table functions exist Table functions are designed as an alternative to final-layer views for cases where views don't push predicates efficiently. A view over `fact_orders` with a `WHERE customer_id = ?` filter may scan the entire table if the engine doesn't push the predicate down. A table function accepts the filter as an argument and guarantees the predicate is applied at execution time. @@ -102,7 +152,7 @@ SELECT customer_id, order_id FROM __table_fn("table_fn__customer_orders")(42) ``` -The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. +The name must be double quoted and the second argument list is required, including for a zero-argument function. SQLBuild validates that the target exists, is a table function, and receives the declared number of arguments. A Python table function call also needs `OVER (PARTITION BY ...)`. The reference creates a typed dependency in the project DAG. Selecting a consuming model also selects the required function, function changes propagate to consumers, and cycles through models and functions are rejected. SQLBuild treats the call as an opaque relation for SQL analysis and uses the function's declared `returns table(...)` columns for star expansion and terminal lineage. @@ -148,19 +198,20 @@ functions/ udf__is_completed_order.sql # scalar SQL UDF table_fn__customer_orders.sql # table function (returns table) python/ - is_completed_order_py.py # scalar Python UDF + is_completed_order_py.py # scalar Python UDF (@udf) + smooth_payments.py # Python table function (@table_fn) ``` -## Python UDF options +## Python function options -The `@udf` decorator accepts these keyword arguments: +The `@udf` and `@table_fn` decorators accept these keyword arguments: | Argument | Required | Description | |----------|----------|-------------| | `description` | Yes, unless the function has a docstring | What the function returns; overrides the docstring | | `arguments` | Yes | Dict mapping argument names to SQL types | -| `returns` | Yes | SQL return type string | -| `runtime_version` | No | Python runtime version (e.g. `"3.11"`) | +| `returns` | Yes | `@udf`: SQL return type string. `@table_fn`: dict mapping each output column to its SQL type | +| `runtime_version` | Yes | Python runtime version (e.g. `"3.11"`) | | `entry_point` | No | Function name to use as the entry point (defaults to the decorated function name) | | `packages` | No | List of Python packages required at runtime | @@ -168,12 +219,11 @@ The `@udf` decorator accepts these keyword arguments: ## Adapter support -All four supported adapters implement SQL UDFs, Python UDFs, and table functions: - -| Feature | DuckDB | Snowflake | BigQuery | Databricks | -|---------|--------|-----------|----------|------------| -| SQL UDFs | Yes | Yes | Yes | Yes | -| Python UDFs | Yes | Yes | Yes | Yes | -| Table functions | Yes | Yes | Yes | Yes | +| Feature | DuckDB, MotherDuck | Snowflake | BigQuery | Databricks | PostgreSQL | SQL Server | ClickHouse | SQLite | +|---------|--------------------|-----------|----------|------------|------------|------------|------------|--------| +| SQL UDFs | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | +| Python UDFs | Yes | Yes | Yes | Yes | No | No | No | No | +| SQL table functions | Yes | Yes | Yes | Yes | No | No | No | No | +| Python table functions | No | Yes | No | No | No | No | No | No | -Future adapters may not support all function types. SQLBuild raises a clear error if a function type is unsupported by the configured adapter. +Creating a function the adapter doesn't support fails with an error naming the adapter and the function type. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 03f4dcd7bd..9fdfd6d64a 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -392,6 +392,9 @@ instead. - Placement: `database`, `schema`, `seed_database`, `seed_schema`, `function_database`, `function_schema` +- Functions: `function_dataframe`, the frame library of Python table functions whose parameter + names none: `"polars"` (default) or `"pandas"` (see + [Python table functions](/docs/concepts/functions#python-table-functions)) - Materialization: `materialized`, `contract`, `full_refresh`, `replay_on_change`, `tags` - Incremental: `incremental_strategy`, `incremental_mode`, `merge_exclude_columns`, `append_cursor_inclusive`, `lookback`, `unaccounted_partition_policy` diff --git a/website/src/content/docs/docs/concepts/testing-reference.mdx b/website/src/content/docs/docs/concepts/testing-reference.mdx index bc243cda2d..50e1c2eaaa 100644 --- a/website/src/content/docs/docs/concepts/testing-reference.mdx +++ b/website/src/content/docs/docs/concepts/testing-reference.mdx @@ -179,8 +179,8 @@ __table_fn_expected__ AS ( ) ``` -Table function tests run after the function is created. Models can't depend on table functions, so -these tests check the function on its own. +Table function tests run after the function is created. Call a Python table function with +`OVER (PARTITION BY ...)` in the test, as in a model. ## Repeating test logic