Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
SHELL := /bin/bash

.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \
.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-parse-guard check-dupes-new check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client \
check-e2e-shards check-test-shards test-shard-core test-shard-publication test-shard-history \
test-e2e-duckdb test-e2e-duckdb-build-core \
test-e2e-duckdb-build-incremental \
Expand Down Expand Up @@ -747,7 +747,7 @@ verify-quick:
exit $$status


check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint
check-ci: check-dupes-new check-e2e-shards check-test-shards check-native-layering check-parse-guard check-native-stub check-build-features check-python-boundary check-release-gate check-docs-skill check-control-client rust-lint
uv run ruff format --check .
uv run ruff check .
uv run ty check src tests
Expand All @@ -770,6 +770,10 @@ check-native-stub:
uv run python -m scripts.check_native_stub


check-parse-guard:
uv run python -m scripts.check_parse_guard


check-build-features:
uv run python -m scripts.check_build_features

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::num::IntErrorKind;
use sqlquack_parse::{StrictFailure, parse_strict};
use sqlquack_semantics::dialects::constants::{SNOWFLAKE, SNOWFLAKE_QUOTED_IGNORING_CASE};
use sqlquack_semantics::dialects::types::DialectProfile;
use sqlquack_syntax::{Dialect, Error, Expression, ParseOptions, Parsed, TokenizeFailure};
use sqlquack_syntax::{Dialect, Error, Expression, Parsed, TokenizeFailure};

use crate::analysis_route::_helpers::dialects::engine_types::snowflake_engine_type;
use crate::analysis_route::_helpers::dialects::type_spelling::respelt;
Expand All @@ -14,6 +14,7 @@ use crate::analysis_route::constants::{
};
use crate::analysis_route::models::{AnalysisInput, ParserGap, SnowflakeDialect};
use crate::analysis_route::types::NativeDialect;
use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options;
use crate::type_system::main::normalize_text_type::normalize_text_type;
use crate::type_system::models::{NormalizedType, TypeDialect, TypeFamily};

Expand Down Expand Up @@ -67,7 +68,7 @@ impl NativeDialect for SnowflakeDialect {

/// Every statement the shared parser reads from `sql`, or its first refusal, never marked syntax.
pub(crate) fn shared_parsed(sql: &str, dialect: Dialect) -> Result<Parsed, ParserGap> {
let statements: Vec<Expression> = parse_strict(sql, dialect, &ParseOptions::default())
let statements: Vec<Expression> = parse_strict(sql, dialect, &sqlquack_parse_options())
.map_err(|failure| match failure {
StrictFailure::Tokens(
TokenizeFailure::Uncovered { start, .. } | TokenizeFailure::Overlap { start, .. },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,20 @@

use polyglot_sql::expressions::Cte;
use polyglot_sql::traversal::ExpressionWalk;
use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions, ast_json};
use serde_json::{Map, Value, json};
use polyglot_sql::{DataType, Dialect, Expression, ParseOptions, ast_json};
use serde_json::{Map, Value};

use crate::assembly::analysis_session::models::ProjectionFacts;
use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options;
use crate::relation_markers::constants::MARKER_NAME_PREFIX;

use crate::assembly::analysis_session::constants::{
ALIAS_AST_KIND, ANNOTATED_AST_KIND, ARRAY_FIXTURE_TYPE, ARRAY_PAYLOAD_KIND, CATALOG_TABLE_ARG,
COLUMN_AST_KIND, CTE_ALIAS_PAYLOAD_KEY, CTE_LIST_PAYLOAD_KEY, DEPENDENCY_FUNCTION_NAMES,
FUNCTION_AST_KIND, FUNCTION_FIXTURE_TYPES, FUNCTION_PAYLOAD_KIND, HELPER_READS_DIALECT,
LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, MAX_FUNCTION_CALL_DEPTH, NAME_PAYLOAD_KEY,
NULL_PAYLOAD_KIND, RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY,
SET_OPERATION_PAYLOAD_KEYS, STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND,
VARIANT_FIXTURE_TYPE, WILDCARD,
LITERAL_FIXTURE_TYPES, LITERAL_PAYLOAD_KIND, NAME_PAYLOAD_KEY, NULL_PAYLOAD_KIND,
RENDERED_TYPE_NAMES, SCHEMA_TABLE_ARG, SELECT_PAYLOAD_KEY, SET_OPERATION_PAYLOAD_KEYS,
STAR_AST_KIND, STRUCTURED_PAYLOAD_KINDS, TABLE_AST_KIND, VARIANT_FIXTURE_TYPE, WILDCARD,
};

/// Python's `bool(value)`.
Expand Down Expand Up @@ -193,12 +193,7 @@ pub(crate) fn parse_one_or_error(
sql: &str,
dialect: &str,
) -> Result<Result<Expression, String>, String> {
let guard: ComplexityGuardOptions =
serde_json::from_value(json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH}))
.map_err(|error| error.to_string())?;
let options: ParseOptions = ParseOptions {
complexity_guard: Some(guard),
};
let options: ParseOptions = polyglot_parse_options();
let parsed: Result<Vec<Expression>, _> = Dialect::get_by_name(dialect)
.ok_or_else(|| format!("unknown dialect {dialect}"))?
.parse_with_options(sql, &options);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use serde_json::{Map, Value};
use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic;
use sqlbuild_core::panics::main::native_failure::native_failure;
use sqlbuild_sqltext::sql_scan::main::dialect_lexical_syntax::dialect_lexical_syntax;
use sqlquack_syntax::{CallMode, Dialect, Expression, ParseOptions};
use sqlquack_syntax::{CallMode, Dialect, Expression};

use crate::assembly::analysis_session::_helpers::dict_walk::{
casefold, column_name, dict_list, identifier_name, is_single_wildcard, nested, node_key,
Expand All @@ -25,6 +25,7 @@ use crate::assembly::analysis_session::models::{
ColumnFact, ContractProof, DynamicFamily, PivotModel, PivotOutcome, PivotTables,
};
use crate::assembly::analysis_session::types::Shapes;
use crate::parse_guard::main::sqlquack_parse_options::sqlquack_parse_options;
use crate::relation_markers::constants::MARKER_NAME_PREFIX;
use crate::relation_markers::main::markers_as_names::markers_as_names;
use crate::relation_markers::types::MarkerCall;
Expand Down Expand Up @@ -134,7 +135,7 @@ fn parsed(sql: &str, dialect: &str) -> Result<Result<Value, String>, String> {
let parsed = sqlquack_parse::parse_statements_with(
&named,
dialect,
&ParseOptions::default(),
&sqlquack_parse_options(),
CallMode::Faithful,
);
let mut statements: Vec<Expression> = match parsed {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,15 @@ use crate::assembly::analysis_session::_helpers::dict_walk::{truthy, upper};
use crate::assembly::analysis_session::_helpers::mappings::ShapeTable;
use crate::assembly::analysis_session::constants::{
CAST_TRANSFORM, CONFIDENCE_HIGH, CONFIDENCE_MEDIUM, CONFIDENCE_UNKNOWN, FILTER_CONTEXT,
MAX_FUNCTION_CALL_DEPTH, NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE,
SELECT_SHAPE, SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT,
NON_NULL_NULLABILITY, NULL_KEYWORD, RESOLVED_SOURCE_CONFIDENCE, SELECT_SHAPE,
SET_OPERATION_SHAPE, TRANSFORM_AGGREGATION, TRANSFORM_CAST, TRANSFORM_CONSTANT,
TRANSFORM_DIRECT, TRANSFORM_EXPRESSION, TRANSFORM_STAR, UNKNOWN_NULLABILITY, UNKNOWN_TYPE,
WILDCARD,
};
use crate::assembly::analysis_session::models::NullabilityCallback;
use crate::assembly::analysis_session::models::{ColumnFact, LineageRow, ModelRequest};
use crate::assembly::analysis_session::types::{Pairs, Shapes};
use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard;
use crate::semantic_validation::main::normalize::normalize_analysis_sql;
use crate::semantic_validation::models::NormalizationInput;

Expand Down Expand Up @@ -75,12 +76,9 @@ fn analysed(input: &EnrichmentInput<'_>) -> Result<Enrichment, String> {
if let Some(schema) = analysis_schema(input) {
options.insert("schema".to_owned(), schema);
}
options.insert(
"complexityGuard".to_owned(),
json!({"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH}),
);
let options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options))
let mut options: AnalyzeQueryOptions = serde_json::from_value(Value::Object(options))
.map_err(|error| format!("query analysis options: {error}"))?;
options.complexity_guard = Some(polyglot_complexity_guard());
let analysis = match analyze_query(&cleaned, options) {
Ok(analysis) => analysis,
Err(_) => return legacy(input, &cleaned),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,6 @@ pub(crate) const FILTER_CONTEXT: &str = "filter";
pub(crate) const NULL_KEYWORD: &str = "NULL";
pub(crate) const RESOLVED_SOURCE_CONFIDENCE: &str = "resolved";
pub(crate) const WILDCARD: &str = "*";
/// SQLBuild's `analyze_query` complexity guard.
pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512;
/// The dialects whose dynamic pivots the compiler proves, and those with DuckDB's PIVOT form.
pub(crate) const SUPPORTED_PIVOT_DIALECTS: [&str; 3] = ["duckdb", "motherduck", "snowflake"];
pub(crate) const SIMPLIFIED_PIVOT_DIALECTS: [&str; 2] = ["duckdb", "motherduck"];
Expand Down
13 changes: 4 additions & 9 deletions crates/sqlbuild-analysis/src/assembly/project/_helpers/syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@

use std::collections::HashMap;

use polyglot_sql::{
ComplexityGuardOptions, Dialect, DialectType, ParseOptions, ValidationOptions,
validate_with_dialect,
};
use polyglot_sql::{Dialect, DialectType, ParseOptions, ValidationOptions, validate_with_dialect};

use crate::assembly::project::constants::{INVALID_SQL, MAX_FUNCTION_CALL_DEPTH};
use crate::assembly::project::constants::INVALID_SQL;
use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard;
use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults;

use crate::assembly::project::models::{SyntaxCheck, SyntaxFailure, SyntaxMode};
Expand Down Expand Up @@ -40,10 +38,7 @@ pub(crate) fn syntax_error(
};
let dialect_type: DialectType = parser_dialect(Some(dialect))
.map_err(|unknown| SyntaxFailure::UnknownDialect(unknown.to_owned()))?;
let complexity_guard = ComplexityGuardOptions {
max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH),
..Default::default()
};
let complexity_guard = polyglot_complexity_guard();
let polyglot: Dialect = Dialect::get(dialect_type);
Ok(match mode {
SyntaxMode::Validate => {
Expand Down
1 change: 0 additions & 1 deletion crates/sqlbuild-analysis/src/assembly/project/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${";
pub(crate) const PYTHON_NONE: &str = "None";
pub(crate) const PYTHON_TRUE: &str = "True";
pub(crate) const PYTHON_FALSE: &str = "False";
pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512;
pub(crate) const MODEL_NAME_CONTEXT: &str = "model.name";
pub(crate) const MODEL_DATABASE_CONTEXT: &str = "model.database";
pub(crate) const MODEL_SCHEMA_CONTEXT: &str = "model.schema";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Resolve every reference to one relation column, with the spans a rename rewrites.

use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options;
use polyglot_sql::{
Dialect, DialectType, Expression, MappingSchema, Resolver, Scope, ValidationSchema,
build_scope, mapping_schema_from_validation_schema_with_dialect, walk_in_scope,
Expand Down Expand Up @@ -109,7 +110,9 @@ pub(crate) fn analyze_json(request_json: &str) -> Result<String, String> {
request.dialect
)
})?;
let Ok(statements) = Dialect::get(dialect).parse(&request.sql) else {
let Ok(statements) =
Dialect::get(dialect).parse_with_options(&request.sql, &polyglot_parse_options())
else {
return serde_json::to_string(&ReferenceResponse::default())
.map_err(|error| error.to_string());
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

use polyglot_sql::Dialect;

use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options;
use crate::compiler::constants::FUNCTION_TYPE_DIALECTS;
use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options;

/// The error for a type the adapter's dialect cannot parse; adapters without a dialect pass.
pub(crate) fn function_type_error(
Expand All @@ -17,9 +17,8 @@ pub(crate) fn function_type_error(
else {
return Ok(None);
};
let options = guarded_parse_options().map_err(|error| error.to_string())?;
Ok(Dialect::get(*dialect)
.parse_data_type_with_options(type_sql, &options)
.parse_data_type_with_options(type_sql, &polyglot_parse_options())
.err()
.map(|error| {
format!(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
pub(crate) mod function_types;
pub(crate) mod parse_options;

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use polyglot_sql::tokens::{Token, TokenType};
use polyglot_sql::{Dialect, DialectType, Expression};

use crate::contracts::models::LiteralLocation;
use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options;

/// How deep the walk follows set operations and passthrough columns before it stops.
const MAX_DEPTH: usize = 64;
Expand Down Expand Up @@ -69,7 +70,9 @@ pub(crate) fn parsed_query(sql: &str, dialect: &str) -> Option<Expression> {
let Ok(dialect) = dialect.parse::<DialectType>() else {
return None;
};
let Ok(mut statements) = polyglot_sql::parse(sql, dialect) else {
let Ok(mut statements) =
polyglot_sql::parse_with_options(sql, dialect, &polyglot_parse_options())
else {
return None;
};
(statements.len() == 1).then(|| statements.pop()).flatten()
Expand Down
1 change: 0 additions & 1 deletion crates/sqlbuild-analysis/src/diff_selection/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,3 @@ pub(crate) const UNKNOWN_TAG_CODE: &str = "S008";
pub(crate) const UNKNOWN_PATH_CODE: &str = "S009";
pub(crate) const RAW_QUERY_READ_ONLY_ROOTS: [&str; 5] =
["except", "intersect", "select", "union", "values"];
pub(crate) const RAW_QUERY_GUARD: &str = r#"{"maxFunctionCallDepth": 512}"#;
Original file line number Diff line number Diff line change
@@ -1,25 +1,19 @@
//! Parse one raw diff query as the Polyglot wheel's guarded `parse_one` does and read its shape.

use polyglot_sql::Expression;
use polyglot_sql::dialects::Dialect;
use polyglot_sql::traversal::ExpressionWalk;
use polyglot_sql::{ComplexityGuardOptions, Expression, ParseOptions};

use crate::diff_selection::constants::{RAW_QUERY_GUARD, RAW_QUERY_READ_ONLY_ROOTS};
use crate::diff_selection::constants::RAW_QUERY_READ_ONLY_ROOTS;
use crate::diff_selection::models::RawQueryShape;
use crate::parse_guard::main::polyglot_parse_options::polyglot_parse_options;

/// The query's shape, or the text of the exception the wheel raises for it.
pub fn raw_query_shape(sql: &str, dialect: &str) -> Result<RawQueryShape, String> {
let parser: Dialect =
Dialect::get_by_name(dialect).ok_or_else(|| format!("Unknown dialect: {dialect}"))?;
let guard: ComplexityGuardOptions =
serde_json::from_str(RAW_QUERY_GUARD).map_err(|error| error.to_string())?;
let mut statements: Vec<Expression> = parser
.parse_with_options(
sql,
&ParseOptions {
complexity_guard: Some(guard),
},
)
.parse_with_options(sql, &polyglot_parse_options())
.map_err(|error| error.to_string())?;
if statements.len() != 1 {
return Err(format!("Expected 1 statement, found {}", statements.len()));
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ pub mod lineage_graph;
pub mod manifest;
pub mod node_set_dependencies;
pub mod nullability_rules;
pub mod parse_guard;
pub mod planner_changes;
pub mod planner_fixtures;
pub mod planner_identity;
Expand Down
21 changes: 4 additions & 17 deletions crates/sqlbuild-analysis/src/lineage/_helpers/parsed_lineage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,37 +2,27 @@

use std::collections::{HashMap, HashSet};

use polyglot_sql::{ComplexityGuardOptions, Dialect, DialectType, Expression, ParseOptions};
use polyglot_sql::{Dialect, DialectType, Expression, ParseOptions};

use crate::lineage::_helpers::projections::{
UnreadableExpression, classify_transform, is_star, projection_output_name,
projection_upstream_columns, set_expression_selects, table_alias_map, unaliased,
};
use crate::lineage::_helpers::references::{PhysicalResource, normalized_sql, physical_resources};
use crate::lineage::_helpers::stars::star_lineage;
use crate::lineage::constants::{KIND_SELECT, KIND_UNION, MAX_FUNCTION_CALL_DEPTH};
use crate::lineage::constants::{KIND_SELECT, KIND_UNION};
use crate::lineage::models::{
FastLineageOutcome, LineageColumn, LineageConfidence, LineageResourceType, LineageSource,
LineageTransformKind,
};

/// The parse options SQLBuild's Polyglot proxy gives `parse_one`, decoded as the wheel does.
pub(crate) fn proxy_parse_options() -> Result<ParseOptions, serde_json::Error> {
let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({
"maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH,
}))?;
Ok(ParseOptions {
complexity_guard: Some(guard),
})
}

pub(crate) struct ParsedModel<'a> {
pub(crate) query_sql: &'a str,
pub(crate) inferred_names: &'a [String],
pub(crate) schema: &'a HashMap<String, Vec<String>>,
/// The parser dialect, or the dialect name Polyglot does not know.
pub(crate) dialect: Result<DialectType, &'a str>,
pub(crate) options: &'a Result<ParseOptions, serde_json::Error>,
pub(crate) options: &'a ParseOptions,
}

/// The model's lineage outcome; fails on a tree the lineage walk cannot read.
Expand All @@ -41,10 +31,7 @@ pub(crate) fn parsed_model_lineage(model: &ParsedModel<'_>) -> Result<FastLineag
Ok(dialect) => dialect,
Err(name) => return Ok(FastLineageOutcome::UnknownDialect(name.to_owned())),
};
let options: &ParseOptions = model
.options
.as_ref()
.map_err(|_| "the parse options do not build".to_owned())?;
let options: &ParseOptions = model.options;
let physical = physical_resources(model.query_sql);
let sql = normalized_sql(model.query_sql);
let mut statements = match Dialect::get(dialect).parse_with_options(&sql, options) {
Expand Down
8 changes: 3 additions & 5 deletions crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@ use crate::lineage::_helpers::references::{
PhysicalResource, normalized_sql, physical_resource_name, physical_resources,
};
use crate::lineage::_helpers::stars::star_lineage;
use crate::lineage::constants::{MAX_FUNCTION_CALL_DEPTH, STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE};
use crate::lineage::constants::{STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE};
use crate::lineage::models::{
LineageColumn, LineageConfidence, LineageNullability, LineageResourceType,
LineageSchemaResource, LineageSource, LineageTransformKind, RichLineageColumn,
RichLineageOutcome, RichSchemaResource,
};
use crate::parse_guard::main::polyglot_complexity_guard::polyglot_complexity_guard;

/// What every model of one request shares: the dialect, the guard and the known tables.
pub(crate) struct RichContext {
Expand All @@ -42,10 +43,7 @@ impl RichContext {
) -> Self {
Self {
dialect,
guard: ComplexityGuardOptions {
max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH),
..ComplexityGuardOptions::default()
},
guard: polyglot_complexity_guard(),
tables: schema_tables(schema, referenced),
names,
}
Expand Down
Loading
Loading